diff --git a/cli/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/cli/CliSnippetProvider.kt b/cli/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/cli/CliSnippetProvider.kt index b472b2e27..0ca08039d 100644 --- a/cli/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/cli/CliSnippetProvider.kt +++ b/cli/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/cli/CliSnippetProvider.kt @@ -3,61 +3,23 @@ package io.github.kdroidfilter.seforimlibrary.cli import io.github.kdroidfilter.seforimlibrary.dao.repository.SeforimRepository import io.github.kdroidfilter.seforimlibrary.search.LineSnippetInfo import io.github.kdroidfilter.seforimlibrary.search.SnippetProvider +import io.github.kdroidfilter.seforimlibrary.search.SnippetSource +import io.github.kdroidfilter.seforimlibrary.search.SnippetSources import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext -import org.jsoup.Jsoup -import org.jsoup.safety.Safelist /** * CLI implementation of [SnippetProvider] that fetches line content from the database. */ -private const val SNIPPET_NEIGHBOR_WINDOW = 4 -private const val SNIPPET_MIN_LENGTH = 280 - class CliSnippetProvider( private val repository: SeforimRepository, ) : SnippetProvider { - override suspend fun getSnippetSources(lines: List): Map { + override suspend fun getSnippetSources(lines: List): Map { if (lines.isEmpty()) return emptyMap() - return withContext(Dispatchers.IO) { - val byBook = lines.groupBy { it.bookId } - val result = mutableMapOf() - - for ((bookId, bookLines) in byBook) { - val minIdx = bookLines.minOf { it.lineIndex } - val maxIdx = bookLines.maxOf { it.lineIndex } - val rangeStart = (minIdx - SNIPPET_NEIGHBOR_WINDOW).coerceAtLeast(0) - val rangeEnd = maxIdx + SNIPPET_NEIGHBOR_WINDOW - - val allLines = repository.getLines(bookId, rangeStart, rangeEnd) - - val plainByIndex = allLines.associate { line -> - line.lineIndex to cleanHtml(line.content) - } - - for (info in bookLines) { - val basePlain = plainByIndex[info.lineIndex].orEmpty() - val snippetSource = if (basePlain.length >= SNIPPET_MIN_LENGTH) { - basePlain - } else { - val start = (info.lineIndex - SNIPPET_NEIGHBOR_WINDOW).coerceAtLeast(0) - val end = info.lineIndex + SNIPPET_NEIGHBOR_WINDOW - (start..end) - .mapNotNull { plainByIndex[it] } - .joinToString(" ") - } - result[info.lineId] = snippetSource - } + SnippetSources.build(lines) { bookId, from, to -> + repository.getLines(bookId, from, to).associate { it.lineIndex to it.content } } - - result } } - - private fun cleanHtml(content: String): String = - Jsoup - .clean(content, Safelist.none()) - .replace("\\s+".toRegex(), " ") - .trim() } diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 773d830af..38e9b5ef9 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -5,7 +5,7 @@ kotlin = "2.3.20" agp = "9.1.1" jvmToolchain = "25" lucene = "10.5.2" -seforimSemantic = "0.4.0" +seforimSemantic = "0.4.1" onnxruntime = "1.30.0" maven-publish = "0.36.0" kotlinx-coroutines = "1.10.2" diff --git a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/HebrewTextUtils.kt b/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/HebrewTextUtils.kt index 0ad4ec92a..f8cdc53f0 100644 --- a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/HebrewTextUtils.kt +++ b/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/HebrewTextUtils.kt @@ -54,12 +54,22 @@ object HebrewTextUtils { * @param text The input text * @return Text with final letters replaced */ - fun replaceFinalsWithBase(text: String): String = text - .replace('\u05DA', '\u05DB') // ך -> כ - .replace('\u05DD', '\u05DE') // ם -> מ - .replace('\u05DF', '\u05E0') // ן -> נ - .replace('\u05E3', '\u05E4') // ף -> פ - .replace('\u05E5', '\u05E6') // ץ -> צ + fun replaceFinalsWithBase(text: String): String { + if (text.none { it != baseLetter(it) }) return text + val sb = StringBuilder(text.length) + for (c in text) sb.append(baseLetter(c)) + return sb.toString() + } + + /** [c] with a final letter (sofit) as its base form: ך כ, ם מ, ן נ, ף פ, ץ צ. */ + fun baseLetter(c: Char): Char = when (c) { + '\u05DA' -> '\u05DB' + '\u05DD' -> '\u05DE' + '\u05DF' -> '\u05E0' + '\u05E3' -> '\u05E4' + '\u05E5' -> '\u05E6' + else -> c + } /** * Checks if a character is a Hebrew diacritic (nikud or teamim). @@ -71,7 +81,10 @@ object HebrewTextUtils { val code = c.code return (code in 0x0591..0x05AF) || // teamim (code in 0x05B0..0x05BD) || // nikud + meteg - (c == '\u05C1') || (c == '\u05C2') || (c == '\u05C7') + (c == '\u05BF') || // rafe + (c == '\u05C1') || (c == '\u05C2') || // shin and sin dots + (c == '\u05C4') || (c == '\u05C5') || // upper and lower dots + (c == '\u05C7') } /** diff --git a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/LuceneSearchEngine.kt b/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/LuceneSearchEngine.kt index 78afbaa59..77931295e 100644 --- a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/LuceneSearchEngine.kt +++ b/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/LuceneSearchEngine.kt @@ -38,8 +38,6 @@ import org.apache.lucene.util.QueryBuilder import org.apache.lucene.store.FSDirectory import org.apache.lucene.store.NIOFSDirectory import org.apache.lucene.document.IntPoint -import org.jsoup.Jsoup -import org.jsoup.safety.Safelist import java.nio.file.Path import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.ExecutorService @@ -59,6 +57,8 @@ private const val MAX_SYNONYM_BOOST_TERMS: Int = 256 private const val SNIPPET_NEIGHBOR_WINDOW = 4 private const val SNIPPET_MIN_LENGTH = 280 private const val FIELD_BOOK_ID = "book_id" +private const val DEFAULT_NEAR = 5 +private const val PLAN_CACHE_SIZE = 16 private const val FIELD_ANCESTOR_CATEGORY_IDS = "ancestor_category_ids" // Searches run by warmUp(): one and several words, common and rarer ones, a quoted phrase, the divine name (whose // dictionary entry is the largest) @@ -258,7 +258,7 @@ class LuceneSearchEngine( baseBookOnly: Boolean ): SearchSession? { val context = buildSearchContext(query, near, bookFilter, categoryFilter, bookIds, lineIds, baseBookOnly) ?: return null - return LuceneSearchSession(context.query, context.snippetTerms, acquireSearcher()) + return LuceneSearchSession(context.query, context, acquireSearcher()) } override suspend fun attachSnippets(hits: List, query: String, near: Int): List { @@ -266,8 +266,8 @@ class LuceneSearchEngine( val context = buildSearchContext(query, near, null, null, null, null) ?: return hits val sources = withContext(Dispatchers.IO) { snippetSources(hits.map { LineSnippetInfo(it.lineId, it.bookId, it.lineIndex) }) } return hits.map { hit -> - val raw = sources[hit.lineId] ?: hit.rawText - hit.copy(snippet = SnippetBuilder.build(raw, context.snippetTerms), rawText = raw) + val source = sources[hit.lineId] ?: SnippetSource(hit.rawText, 0, hit.rawText.length) + hit.copy(snippet = SnippetBuilder.build(source.text, context.highlightPlan, source.lineStart, source.lineEnd), rawText = source.text) } } @@ -310,27 +310,51 @@ class LuceneSearchEngine( } } - override fun buildSnippet(rawText: String, query: String, near: Int): String { - val parsed = SearchQueryParser.parse(query) - val norm = HebrewTextUtils.normalizeHebrew(parsed.freeText) - val exactPhrasesNorm = parsed.exactPhrases - .map { HebrewTextUtils.normalizeHebrew(it) } - .filter { it.isNotBlank() } - if (norm.isBlank() && exactPhrasesNorm.isEmpty()) return Jsoup.clean(rawText, Safelist.none()) - val rawClean = Jsoup.clean(rawText, Safelist.none()) - val analyzedStd = (analyzeToTerms(stdAnalyzer, norm) ?: emptyList()) - val hasHashem = query.contains("ה׳") || query.contains("ה'") - val hashemTerms = if (hasHashem) loadHashemHighlightTerms() else emptyList() - // Verbatim tokens from the quoted phrases must be highlighted too (no expansion). - val exactPhraseTokens = exactPhrasesNorm.flatMap { analyzeToTerms(stdAnalyzer, it) ?: emptyList() } - val highlightTerms = filterTermsForHighlight( - analyzedStd + buildNgramTerms(analyzedStd, gram = 4) + hashemTerms + exactPhraseTokens - ) - val anchorBasis = (sequenceOf(norm) + exactPhrasesNorm.asSequence()) + override fun buildSnippet(rawText: String, query: String, near: Int): String = + SnippetBuilder.build(SnippetSources.clean(rawText), planFor(query, near)) + + override fun highlightRanges(text: String, query: String): List = + mergedRanges(text, planFor(query, DEFAULT_NEAR).highlights(text)) + + override fun rangeSnippet(text: String, ranges: List): String = + SnippetBuilder.build(text, ranges.map { Occurrence(it.first, it.last + 1, 0, EvidenceLevel.LITERAL) }) + + // The highlight plans of the latest queries: a results page and its preview ask for the same one many times + private val planCache = object : LinkedHashMap(PLAN_CACHE_SIZE, 0.75f, true) { + override fun removeEldestEntry(eldest: MutableMap.MutableEntry?): Boolean = + size > PLAN_CACHE_SIZE + } + + private fun planFor(query: String, near: Int): HighlightPlan { + synchronized(planCache) { planCache[query]?.let { return it } } + val plan = buildSearchContext(query, near, null, null, null, null)?.highlightPlan ?: HighlightPlan.EMPTY + synchronized(planCache) { planCache[query] = plan } + return plan + } + + /** + * What a query highlights: each word with its inflections, from the dictionary entries it truly is a form of and + * that aren't blacklisted for it. With ה׳ in the query, the lone ה is the divine name. + */ + private fun buildHighlightPlan( + tokens: List, + hasHashem: Boolean, + exactPhrases: List + ): HighlightPlan { + val forms = tokens.distinct().map { token -> + if (hasHashem && highlightKey(token) == "ה") { + return@map TokenForms(token, extraLiterals = loadHashemHighlightTerms()) + } + val key = highlightKey(token) + val inflections = magicDict?.expansionsFor(listOf(token)).orEmpty() + .filter { exp -> exp.inflections.any { highlightKey(it) == key } && !isHallucinatedExpansion(token, exp) } + .flatMap { it.inflections } + TokenForms(token, inflections) + } + val phrases = exactPhrases + .map { analyzeToTerms(stdAnalyzer, it).orEmpty().joinToString(" ") } .filter { it.isNotBlank() } - .joinToString(" ") - val anchorTerms = buildAnchorTerms(anchorBasis, highlightTerms) - return SnippetBuilder.build(rawClean, SnippetTerms(anchorTerms, highlightTerms)) + return HighlightPlan.build(forms, phrases) } override fun findInBookCandidates(query: String, bookId: Long): LongArray? { @@ -531,9 +555,9 @@ class LuceneSearchEngine( // --- Inner SearchSession class --- - inner class LuceneSearchSession internal constructor( + private inner class LuceneSearchSession( private val query: Query, - private val snippetTerms: SnippetTerms, + private val context: SearchContext, private val searcher: IndexSearcher ) : SearchSession { private var after: ScoreDoc? = null @@ -549,7 +573,7 @@ class LuceneSearchEngine( return@withContext null } val stored = searcher.storedFields() - val hits = mapScoreDocs(stored, top.scoreDocs.asList(), snippetTerms, snippets) + val hits = mapScoreDocs(stored, top.scoreDocs.asList(), context, snippets) after = top.scoreDocs.last() val isLast = top.scoreDocs.size < limit if (isLast) finished = true @@ -586,10 +610,9 @@ class LuceneSearchEngine( // The required clauses only: the same matches as [query], without the ranking-only clauses (fuzzy, n-grams, // synonym boosts) that cost a rewrite but can't change the hit set val matchQuery: Query, - val anchorTerms: List, - val highlightTerms: List + private val plan: Lazy ) { - val snippetTerms by lazy { SnippetTerms(anchorTerms, highlightTerms) } + val highlightPlan: HighlightPlan get() = plan.value } private fun buildSearchContext( @@ -648,42 +671,8 @@ class LuceneSearchEngine( } } - // For HIGHLIGHTING, filter out hallucinated expansions to avoid highlighting unrelated words - val tokenExpansionsForHighlight: Map> = - tokenExpansions.mapValues { (token, exps) -> - exps.filter { exp -> - val isHallucination = isHallucinatedExpansion(token, exp) - if (isHallucination) { - logger.d { "[DEBUG] Token '$token' -> BLOCKED for highlight (hallucination): base=${exp.base}" } - } - !isHallucination - } - } - - val allExpansionsForHighlight = tokenExpansionsForHighlight.values.flatten() - // Filter out 2-letter terms from dictionary expansions for highlighting - // (2-letter words should only be highlighted if explicitly in the query) - val expandedTerms = allExpansionsForHighlight - .flatMap { it.surface + it.variants + it.base } - .filter { it.length > 2 } - .distinct() - // Add 4-gram terms used in the query (matches text_ng4 clauses) so highlighting can - // reflect matches that were found via the n-gram branch. - val ngramTerms = buildNgramTerms(analyzedStd, gram = 4) - // For highlighting/snippets, use the actual query tokens plus the concrete - // terms that the search query uses (expansions + n-grams), and if the query - // mentions Hashem explicitly, also include dictionary-based variants of the - // divine name from the lexical DB - val hashemTerms = if (hasHashem) loadHashemHighlightTerms() else emptyList() - // Verbatim tokens from the quoted phrases must be highlighted too (no expansion). - val exactPhraseTokens = exactPhrasesNorm.flatMap { analyzeToTerms(stdAnalyzer, it) ?: emptyList() } - val highlightTerms = filterTermsForHighlight( - analyzedStd + expandedTerms + ngramTerms + hashemTerms + exactPhraseTokens - ) - val anchorBasis = (sequenceOf(norm) + exactPhrasesNorm.asSequence()) - .filter { it.isNotBlank() } - .joinToString(" ") - val anchorTerms = buildAnchorTerms(anchorBasis, highlightTerms) + // What the snippets highlight: built on first use (facets need none) + val highlightPlan = lazy { buildHighlightPlan(analyzedStd, hasHashem, exactPhrasesNorm) } val builder = BooleanQuery.Builder() // Mirrors every required clause of builder (the facets need only the hit set) @@ -741,8 +730,7 @@ class LuceneSearchEngine( return SearchContext( query = finalQuery, matchQuery = matchBuilder.build(), - anchorTerms = anchorTerms, - highlightTerms = highlightTerms + plan = highlightPlan ) } @@ -751,7 +739,7 @@ class LuceneSearchEngine( private suspend fun mapScoreDocs( stored: StoredFields, scoreDocs: List, - snippetTerms: SnippetTerms, + context: SearchContext, snippets: Boolean = true ): List { if (scoreDocs.isEmpty()) return emptyList() @@ -782,11 +770,12 @@ class LuceneSearchEngine( ) } - val snippetSources: Map = + val snippetSources: Map = if (snippets) snippetSources(docMetas.map { LineSnippetInfo(it.lineId, it.bookId, it.lineIndex) }) else emptyMap() val hits = docMetas.map { meta -> - val raw = snippetSources[meta.lineId] ?: meta.indexedRaw + val source = snippetSources[meta.lineId] ?: SnippetSource(meta.indexedRaw, 0, meta.indexedRaw.length) + val raw = source.text val baseScore = meta.sd.score // Calculate boost: lower orderIndex = higher boost (only for base books) @@ -799,7 +788,7 @@ class LuceneSearchEngine( baseScore } - val snippet = if (snippets) SnippetBuilder.build(raw, snippetTerms) else "" + val snippet = if (snippets) SnippetBuilder.build(raw, context.highlightPlan, source.lineStart, source.lineEnd) else "" LineHit( bookId = meta.bookId, bookTitle = meta.bookTitle, @@ -816,7 +805,7 @@ class LuceneSearchEngine( } // Snippet sources: from the provider if available, otherwise none (the indexed text_raw fallback) - private suspend fun snippetSources(lines: List): Map = + private suspend fun snippetSources(lines: List): Map = snippetProvider?.getSnippetSources(lines) ?: emptyMap() private suspend fun doSearch( @@ -833,7 +822,7 @@ class LuceneSearchEngine( val top = searcher.search(context.query, offset + limit) val stored: StoredFields = searcher.storedFields() val sliced = top.scoreDocs.drop(offset) - mapScoreDocs(stored, sliced, context.snippetTerms) + mapScoreDocs(stored, sliced, context) } } } @@ -852,7 +841,7 @@ class LuceneSearchEngine( val top = searcher.search(context.query, offset + limit) val stored: StoredFields = searcher.storedFields() val sliced = top.scoreDocs.drop(offset) - mapScoreDocs(stored, sliced, context.snippetTerms) + mapScoreDocs(stored, sliced, context) } } } @@ -1112,50 +1101,6 @@ class LuceneSearchEngine( return b.build() } - private fun buildAnchorTerms(normQuery: String, analyzedTerms: List): List { - val qTokens = normQuery.split("\\s+".toRegex()) - .map { it.trim() } - .filter { it.isNotEmpty() } - val combined = (qTokens + analyzedTerms.map { it.trimEnd('$') }) - val filtered = filterTermsForHighlight(combined) - if (filtered.isNotEmpty()) return filtered - val qFiltered = filterTermsForHighlight(qTokens) - return qFiltered.ifEmpty { qTokens } - } - - private fun filterTermsForHighlight(terms: List): List { - if (terms.isEmpty()) return emptyList() - - fun useful(t: String): Boolean { - val s = t.trim() - if (s.isEmpty()) return false - if (s.length < 2) return false - if (s.none { it.isLetterOrDigit() }) return false - return true - } - return terms - .map { it.trim() } - .filter { useful(it) } - .distinct() - .sortedByDescending { it.length } - } - - private fun buildNgramTerms(tokens: List, gram: Int = 4): List { - if (gram <= 0) return emptyList() - val out = mutableListOf() - tokens.forEach { t -> - val trimmed = t.trim() - if (trimmed.length >= gram) { - var i = 0 - while (i + gram <= trimmed.length) { - out += trimmed.substring(i, i + gram) - i += 1 - } - } - } - return out.distinct() - } - private fun buildLimitedTermsForToken( token: String, expansions: List diff --git a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/MagicDictionaryIndex.kt b/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/MagicDictionaryIndex.kt index 83dd32494..4fb40b056 100644 --- a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/MagicDictionaryIndex.kt +++ b/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/MagicDictionaryIndex.kt @@ -21,10 +21,16 @@ class MagicDictionaryIndex private constructor( private val norm: (String) -> String, private val dbFile: Path ) { + /** + * The forms a token expands to. [surface] holds every form (inflections, then the dictionary's variants, then + * the base) as the ranking uses them; [inflections] only the base's inflected forms and the base itself: the + * variants also hold loose associations (יום for הוא), so they don't count as the same word. + */ data class Expansion( val surface: List, val variants: List, - val base: List + val base: List, + val inflections: List = emptyList() ) private val url = "jdbc:sqlite:${dbFile.toAbsolutePath()}" @@ -81,8 +87,12 @@ class MagicDictionaryIndex private constructor( } if (matchingBase != null) return matchingBase - // Otherwise, prefer the largest expansion (more terms = more complete paradigm) - return expansions.maxByOrNull { it.surface.size } + // Then the entries the token truly is a form of: one reached only through a dictionary variant (loose + // associations: לעולם to יום) isn't the token's paradigm, however large + val ownEntries = expansions.filter { exp -> exp.inflections.any { it == normalized } } + + // The largest of them (more terms = more complete paradigm) + return ownEntries.ifEmpty { expansions }.maxByOrNull { it.surface.size } } private fun expansionsForToken(token: String): List = lookup(token, async = false).join() @@ -118,7 +128,8 @@ class MagicDictionaryIndex private constructor( val surfaces = (existing.surface + exp.surface).distinct() val variants = (existing.variants + exp.variants).distinct() val base = (existing.base + exp.base).distinct() - mergedByBase[baseId] = Expansion(surfaces, variants, base) + val inflections = (existing.inflections + exp.inflections).distinct() + mergedByBase[baseId] = Expansion(surfaces, variants, base, inflections) } } } @@ -170,7 +181,8 @@ class MagicDictionaryIndex private constructor( val exp = Expansion( surface = allTerms, variants = emptyList(), - base = baseTerms + base = baseTerms, + inflections = (surfaceN + baseTerms).distinct() ) synchronized(baseCache) { diff --git a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/QueryHighlighter.kt b/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/QueryHighlighter.kt new file mode 100644 index 000000000..67cf1fd64 --- /dev/null +++ b/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/QueryHighlighter.kt @@ -0,0 +1,257 @@ +package io.github.kdroidfilter.seforimlibrary.search + +/** How surely a word stands for a query word: its own form, or an inflection of it. */ +internal object EvidenceLevel { + const val LITERAL = 0 + const val INFLECTION = 1 + const val NONE = 2 +} + +/** A word of the query and the forms that count as it (normalized like [HebrewTextUtils.normalizeHebrew]). */ +internal class TokenForms( + val literal: String, + val inflections: Collection = emptyList(), + // The divine name: ה׳, יהוה... (a lone ה stands for it only with its geresh) + val extraLiterals: Collection = emptyList(), +) + +/** A word of a text: its range and its comparison key (no diacritics, quotes or final letters, lowercase). */ +internal class TextWord( + val start: Int, + val end: Int, + val key: String, + // Had a geresh or gershayim (an abbreviation or a number: ה׳, רש״י) + val quoted: Boolean, +) + +/** Where a query unit (a word of the query, or a quoted phrase) shows in a text, and how surely. */ +internal class Occurrence( + val start: Int, + val end: Int, + val unit: Int, + val level: Int, +) + +/** + * What a search highlights in a text: whole words that make a line match the query. A query word counts as itself + * (also behind Hebrew prefixes, or inside a longer word for 4 letters and more, as the n-gram matching allows) or as + * an inflection from the dictionary; never as a dictionary variant, which may be a loose association (עם for כל) the + * search uses to find more but that isn't the word. Quoted phrases count as their exact words in a row. + */ +internal class HighlightPlan private constructor( + private val single: Map>, + private val multi: Map>, + private val substrings: List>, + private val phrases: List, Int>>, + val unitCount: Int, +) { + private class FormRef(val unit: Int, val level: Int, val needsQuote: Boolean) + + private class MultiForm(val rest: List, val unit: Int, val level: Int) + + val isEmpty: Boolean get() = unitCount == 0 + + /** The occurrences of the query units among [words] (from [words] of [text]), sorted by position. */ + fun occurrences(words: List): List { + if (isEmpty || words.isEmpty()) return emptyList() + val out = ArrayList() + val best = IntArray(unitCount) { EvidenceLevel.NONE } + val touched = ArrayList(4) + for ((i, word) in words.withIndex()) { + val key = word.key + fun offer(unit: Int, level: Int) { + if (best[unit] == EvidenceLevel.NONE) touched += unit + if (level < best[unit]) best[unit] = level + } + // The word itself, then without up to three prefix letters + for (p in 0..minOf(MAX_PREFIX, key.length - 1)) { + if (p > 0 && key[p - 1] !in PREFIX_LETTERS) break + val stem = if (p == 0) key else key.substring(p) + if (p > 0 && !prefixAllowed(key, p, stem.length)) continue + single[stem]?.forEach { ref -> + if (!ref.needsQuote || (p == 0 && word.quoted)) offer(ref.unit, ref.level) + } + } + for ((literal, unit) in substrings) { + if (key.length > literal.length && key.contains(literal)) offer(unit, EvidenceLevel.LITERAL) + } + for (unit in touched) { + out += Occurrence(word.start, word.end, unit, best[unit]) + best[unit] = EvidenceLevel.NONE + } + touched.clear() + // Dictionary forms of several words + multi[key]?.forEach { form -> + if (matchesRun(words, i + 1, form.rest)) { + out += Occurrence(word.start, words[i + form.rest.size].end, form.unit, form.level) + } + } + } + for ((phrase, unit) in phrases) { + for (i in 0..words.size - phrase.size) { + if (words[i].key == phrase[0] && matchesRun(words, i + 1, phrase.subList(1, phrase.size))) { + out += Occurrence(words[i].start, words[i + phrase.size - 1].end, unit, EvidenceLevel.LITERAL) + } + } + } + out.sortWith(compareBy { it.start }.thenBy { it.end }) + return out + } + + /** The occurrences of the query units in [text], sorted by position. */ + fun highlights(text: String): List = occurrences(splitWords(text)) + + private fun matchesRun(words: List, from: Int, keys: List): Boolean { + if (from + keys.size > words.size) return false + for (k in keys.indices) if (words[from + k].key != keys[k]) return false + return true + } + + companion object { + private const val MAX_PREFIX = 3 + + // ו ה ב כ ל מ ש, and the Aramaic ד + private const val PREFIX_LETTERS = "והבכלמשד" + + // Keys a query word must exceed to be matched inside a longer word, as the 4-gram matching does + private const val MIN_SUBSTRING_LENGTH = 4 + + /** + * A prefix may only be split off a stem of 3 letters or more; a 2-letter stem only takes ו or ה, so that + * כל isn't found in שכל nor בן in לבן. + */ + private fun prefixAllowed(key: String, prefixLength: Int, stemLength: Int): Boolean = + stemLength >= 3 || (stemLength == 2 && prefixLength == 1 && (key[0] == 'ו' || key[0] == 'ה')) + + val EMPTY = HighlightPlan(emptyMap(), emptyMap(), emptyList(), emptyList(), 0) + + fun build(tokens: List, phrases: List): HighlightPlan { + val single = HashMap>() + val multi = HashMap>() + val substrings = ArrayList>() + var unit = 0 + + fun add(form: String, unitIndex: Int, level: Int) { + val keys = form.split(' ').map { highlightKey(it) }.filter { it.isNotEmpty() } + when { + keys.isEmpty() -> return + keys.size == 1 -> { + val key = keys[0] + val refs = single.getOrPut(key) { mutableListOf() } + if (refs.none { it.unit == unitIndex && it.level <= level }) { + refs += FormRef(unitIndex, level, needsQuote = key.length < 2) + } + } + else -> multi.getOrPut(keys[0]) { mutableListOf() } += MultiForm(keys.drop(1), unitIndex, level) + } + } + + for (token in tokens) { + val index = unit++ + add(token.literal, index, EvidenceLevel.LITERAL) + token.extraLiterals.forEach { add(it, index, EvidenceLevel.LITERAL) } + token.inflections.forEach { add(it, index, EvidenceLevel.INFLECTION) } + val key = highlightKey(token.literal) + if (key.length >= MIN_SUBSTRING_LENGTH && ' ' !in token.literal) substrings += key to index + } + val phraseUnits = phrases.mapNotNull { phrase -> + val keys = phrase.split(' ').map { highlightKey(it) }.filter { it.isNotEmpty() } + if (keys.isEmpty()) null else keys to unit++ + } + if (unit == 0) return EMPTY + return HighlightPlan(single, multi, substrings, phraseUnits, unit) + } + } +} + +private fun isQuote(c: Char): Boolean = c == '"' || c == '\'' || c == '״' || c == '׳' + +private fun keyLetter(c: Char): Char = HebrewTextUtils.baseLetter(c).lowercaseChar() + +/** The comparison key of a word or form: no diacritics or quotes, final letters as their base, lowercase. */ +internal fun highlightKey(word: String): String { + val sb = StringBuilder(word.length) + for (c in word) if (!HebrewTextUtils.isNikudOrTeamim(c) && !isQuote(c) && !c.isWhitespace()) sb.append(keyLetter(c)) + return sb.toString() +} + +/** + * Splits [text] into words: letters and digits with their diacritics, gershayim between letters (רש״י) and a + * closing geresh (ה׳, וכו׳) included. HTML entities (the snippet sources are escaped) are skipped, as is anything + * else. + */ +internal fun splitWords(text: String): List { + val out = ArrayList() + var i = 0 + val n = text.length + while (i < n) { + val c = text[i] + if (c == '&') { + val end = entityEnd(text, i) + if (end > i) { + i = end + continue + } + } + if (!c.isLetterOrDigit()) { + i++ + continue + } + val start = i + var quoted = false + val key = StringBuilder() + while (i < n) { + val ch = text[i] + when { + ch.isLetterOrDigit() -> key.append(keyLetter(ch)) + HebrewTextUtils.isNikudOrTeamim(ch) -> Unit + // Gershayim or geresh inside the word: רש״י, ע"ש + isQuote(ch) && i + 1 < n && text[i + 1].isLetter() -> quoted = true + else -> break + } + i++ + } + // A closing geresh: ה׳, וכו' + if (i < n && (text[i] == '׳' || text[i] == '\'')) { + quoted = true + i++ + } + out += TextWord(start, i, key.toString(), quoted) + } + return out +} + +// The end of the HTML entity at [start] (& '  ), or [start] when there is none +private fun entityEnd(text: String, start: Int): Int { + var i = start + 1 + while (i < text.length && i - start <= 10) { + val c = text[i] + if (c == ';') return if (i > start + 1) i + 1 else start + if (!c.isLetterOrDigit() && c != '#') return start + i++ + } + return start +} + +/** + * The ranges of [occurrences] (sorted by start) in [text]: overlapping ones, and ones apart only by spaces or a maqaf, + * as one, so a run of matched words reads as one span rather than a word-by-word patchwork. + */ +internal fun mergedRanges(text: String, occurrences: List): List { + val out = ArrayList() + for (o in occurrences) { + val last = out.lastOrNull() + if (last != null && joins(text, last.last + 1, o.start)) { + out[out.lastIndex] = last.first..maxOf(last.last, o.end - 1) + } else { + out += o.start until o.end + } + } + return out +} + +// Whether nothing but spaces and maqafs lies from [from] to [until] (also when they overlap) +private fun joins(text: String, from: Int, until: Int): Boolean { + for (i in from until until) if (!text[i].isWhitespace() && text[i] != '־') return false + return true +} diff --git a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SearchEngine.kt b/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SearchEngine.kt index 47f42f364..2aa41de91 100644 --- a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SearchEngine.kt +++ b/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SearchEngine.kt @@ -85,6 +85,25 @@ interface SearchEngine : Closeable { */ fun buildSnippet(rawText: String, query: String, near: Int): String + /** + * The words that stand for [query] in [text] (plain, diacritics allowed), as ranges of [text]: the one rule every + * view highlights a search with (results' snippets, their preview, the reader's smart find). A query word counts + * as itself (prefixed, or inside a longer word from 4 letters) or as one of its inflections; consecutive matched + * words form one range. + */ + fun highlightRanges(text: String, query: String): List = emptyList() + + /** + * What to highlight of [query] in found lines [texts] (plain): [highlightRanges] where the query's words show, + * else, for engines that search by meaning, the passage closest in meaning to the query (the whole line when it + * is a single clause). One batch, as the meaning takes a model run. + */ + suspend fun highlights(texts: List, query: String): List = + texts.map { TextHighlights(highlightRanges(it, query)) } + + /** A snippet of [text] (snippet text, see [SnippetSources.clean]) laid out around [ranges], bold. */ + fun rangeSnippet(text: String, ranges: List): String = text + /** * Fills `snippet` and `rawText` of [hits] fetched without snippets (`nextPage(limit, snippets = false)`), so * only the hits actually shown pay for their snippet. Order and other fields are kept. @@ -102,24 +121,9 @@ interface SearchEngine : Closeable { suspend fun warmUp() {} /** - * Returns the contiguous passage within [text] whose meaning is closest to [query], - * for semantic highlighting (using the same dense encoder as semantic search). - * - * The result is a verbatim substring of [text] so callers can locate it with the - * usual diacritic-insensitive matching and highlight that single span — instead of - * scattering dictionary-expanded word matches that don't reflect meaning. - * - * @return the best-matching passage, or null when dense search is unavailable or no - * passage stands out (e.g. the text is a single short clause). - */ - suspend fun semanticSpan(query: String, text: String): String? = null - - /** - * Embedding-based find-in-page within a single book: returns the ids of the lines - * semantically closest to [query] (dense KNN over the index, scoped to [bookId]), - * ordered by relevance. Used by the "smart" find mode — the simple mode matches literal - * words instead. The per-line passage to highlight is computed by the caller via - * [semanticSpan] on the displayed text. Empty when dense search is unavailable. + * Smart find-in-page within a single book: the ids of its lines the search finds for [query] (its words, fused + * with the lines closest in meaning where dense search is available), best first. The simple mode matches literal + * words instead. Each line is highlighted with [highlights]. Empty when dense search is unavailable. */ suspend fun semanticFind(query: String, bookId: Long, limit: Int): List = emptyList() @@ -169,3 +173,14 @@ interface SearchEngine : Closeable { baseBookOnly: Boolean = false ): SearchFacets? } + +/** + * What a search highlights in a found line. + * + * @property ranges the highlighted ranges of the line's text + * @property byMeaning true when they are the passage closest in meaning, none of the query's words being there + */ +data class TextHighlights( + val ranges: List, + val byMeaning: Boolean = false +) diff --git a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SnippetBuilder.kt b/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SnippetBuilder.kt index 07492d096..2f8debb2f 100644 --- a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SnippetBuilder.kt +++ b/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SnippetBuilder.kt @@ -1,263 +1,192 @@ package io.github.kdroidfilter.seforimlibrary.search /** - * The terms of one search, prepared once to build the snippets of all its hits: dictionary expansions make them - * hundreds, so matching them one by one against every text was the main cost of a results page. - * - * @param anchorTerms the terms whose clustering picks the snippet's window (matched anywhere, case-sensitive) - * @param highlightTerms the terms put in bold (whole words only, case-insensitive) - */ -internal class SnippetTerms( - val anchorTerms: List, - highlightTerms: List, -) { - val anchors = AhoCorasick(anchorTerms) - - private val pool = (highlightTerms + highlightTerms.map { it.trimEnd('$') }) - .distinct() - .filter { it.isNotBlank() } - .map { it.lowercase() } - - // A term made of letters and digits only matches whole words exactly: looked up per word of the text - val wordTerms: Set = pool.filterTo(HashSet()) { term -> term.all { it.isLetterOrDigit() } } - - // Any other term (punctuation, spaces) is searched in the text - val otherTerms: List = pool.filter { it !in wordTerms } -} - -/** - * Builds the HTML snippet of [raw] for a search: a window around where most anchor terms cluster, with the - * highlight terms in ``. + * Builds the HTML snippet of a hit: its text from just before where the query's words gather best, with the words + * that stand for them in `` (see [HighlightPlan]). Results show the first lines of a snippet only, so the matches + * lead it, as in a search engine's results page. */ internal object SnippetBuilder { - // Occurrences of a term considered when choosing the window - private const val MAX_OCCURRENCES_PER_TERM = 5 + // Visible characters (diacritics aside) of a snippet, longer for vocalized text + private const val LENGTH = 440 + private const val LENGTH_WITH_DIACRITICS = 720 - fun build(raw: String, terms: SnippetTerms, context: Int = 220): String { - if (raw.isEmpty()) return "" - val (plain, mapToOrig) = HebrewTextUtils.stripDiacriticsWithMap(raw) - val hasDiacritics = plain.length != raw.length - val effContext = if (hasDiacritics) maxOf(context, 360) else context - val plainSearch = HebrewTextUtils.replaceFinalsWithBase(plain) + // Span of a group of matches: what the first two lines of a result show past the lead-in + private const val CLUSTER_REACH = 140 - val (plainIdx, plainLen) = bestAnchor(plainSearch, terms, effContext) - val plainStart = (plainIdx - effContext).coerceAtLeast(0) - val plainEnd = (plainIdx + plainLen + effContext).coerceAtMost(plain.length) - val origStart = HebrewTextUtils.mapToOrigIndex(mapToOrig, plainStart) - val origEnd = HebrewTextUtils.mapToOrigIndex(mapToOrig, plainEnd).coerceAtMost(raw.length) - - val base = raw.substring(origStart, origEnd) - val basePlain = plain.substring(plainStart, plainEnd) - val basePlainSearch = HebrewTextUtils.replaceFinalsWithBase(basePlain) - val baseMap = IntArray(plainEnd - plainStart) { idx -> - (mapToOrig[plainStart + idx] - origStart).coerceIn(0, base.length.coerceAtLeast(1) - 1) - } - val text = basePlainSearch.lowercase() + // Context shown before the first match, at most: from the start of its sentence when that is closer + private const val LEAD = 30 + private const val SENTENCE_ENDS = ".:;?!׃" - val intervals = mutableListOf() - fun addMatch(idx: Int, length: Int) { - val startOrig = HebrewTextUtils.mapToOrigIndex(baseMap, idx) - val endOrig = HebrewTextUtils.mapToOrigIndex(baseMap, idx + length - 1) + 1 - if (startOrig in 0 until endOrig && endOrig <= base.length) intervals += (startOrig until endOrig) - } + // Bounds the window search on texts with a great many matches + private const val MAX_CANDIDATES = 400 - // Whole words: each word of the text looked up among the terms - if (terms.wordTerms.isNotEmpty()) { - var i = 0 - while (i < text.length) { - if (!text[i].isLetterOrDigit()) { - i++ - continue - } - val start = i - while (i < text.length && text[i].isLetterOrDigit()) i++ - if (text.substring(start, i) in terms.wordTerms) addMatch(start, i - start) - } - } - for (term in terms.otherTerms) { - var from = 0 - while (from <= text.length - term.length) { - val idx = text.indexOf(term, startIndex = from) - if (idx == -1) break - if (isWordBoundary(text, idx - 1) && isWordBoundary(text, idx + term.length)) addMatch(idx, term.length) - from = idx + 1 + /** + * @param raw the snippet source, HTML-escaped text (the hit line, possibly with its neighbors) + * @param lineStart start of the hit line in [raw]; the window favors the matches inside it + * @param lineEnd end of the hit line in [raw] + */ + fun build( + raw: String, + plan: HighlightPlan, + lineStart: Int = 0, + lineEnd: Int = raw.length, + ): String = build(raw, plan.highlights(raw), lineStart, lineEnd) + + /** The snippet of [raw] with [occurrences] (sorted by start) as its matches. */ + fun build( + raw: String, + occurrences: List, + lineStart: Int = 0, + lineEnd: Int = raw.length, + ): String { + if (raw.isEmpty()) return "" + val visible = VisibleIndex(raw) + val window = if (visible.hasDiacritics) LENGTH_WITH_DIACRITICS else LENGTH + + val cluster = bestCluster(occurrences, visible, lineStart, lineEnd) + + // From a short lead-in before the cluster, or from the hit line's start without any + val (focusStart, focusEnd) = cluster ?: (lineStart to lineStart) + // (from the hit line's start at the earliest when the cluster is in it) + var start = + when { + cluster == null -> lineStart + focusStart >= lineStart && focusStart < lineEnd -> maxOf(lineStart, leadIn(raw, visible, focusStart)) + else -> leadIn(raw, visible, focusStart) } - } + val endVisible = (visible.at(start) + window).coerceAtMost(visible.total) + var end = if (endVisible >= visible.total) raw.length else visible.offsetOf(endVisible) + + // Cut between words only, never inside the matched ones + if (start > 0) start = snapForward(raw, start, focusStart) + if (end < raw.length) end = snapBackward(raw, end, maxOf(focusEnd, start)) + + val shown = occurrences.filter { it.start >= start && it.end <= end } + val body = withBold(raw, start, end, mergedRanges(raw, shown)) + val prefix = if (start > 0) "..." else "" + val suffix = if (end < raw.length) "..." else "" + return prefix + body.trim() + suffix + } - val merged = mergeIntervals(intervals.sortedBy { it.first }) - val highlighted = insertBoldTags(base, merged) - val prefix = if (origStart > 0) "..." else "" - val suffix = if (origEnd < raw.length) "..." else "" - return prefix + highlighted + suffix + // Where the snippet starts before [matchStart]: right after the closest sentence end within the lead-in, else + // the lead-in's length before it (the text's start when that is near) + private fun leadIn(raw: String, visible: VisibleIndex, matchStart: Int): Int { + val leadStart = visible.at(matchStart) - LEAD + if (leadStart <= 0) return 0 + val from = visible.offsetOf(leadStart) + for (i in matchStart - 1 downTo from) { + if (raw[i] in SENTENCE_ENDS && i + 1 < raw.length && raw[i + 1].isWhitespace()) return i + 1 + } + return from } /** - * The window's anchor (plain index and length): the first occurrence, in term order, around which the most - * distinct anchor terms cluster, a longer term breaking ties; the text start when no term occurs. + * The span (start to end) of the best group of matches that fits the first lines of a result: one starting in + * the hit line when it holds any, covering the most distinct query units, then the surest forms, then the most + * matches; the first one on ties. */ - private fun bestAnchor(plainSearch: String, terms: SnippetTerms, effContext: Int): Pair { - val anchorTerms = terms.anchorTerms - var plainIdx = 0 - var plainLen = anchorTerms.firstOrNull()?.length ?: 0 - if (anchorTerms.isEmpty()) return plainIdx to plainLen - - // The first occurrences of each term, overlapping ones included - val occurrences = terms.anchors.firstOccurrences(plainSearch, MAX_OCCURRENCES_PER_TERM) - val count = occurrences.sumOf { it.size } - if (count == 0) return plainIdx to plainLen - // Candidates in term order, then by position - val candidatePos = IntArray(count) - val candidateTerm = IntArray(count) - var n = 0 - occurrences.forEachIndexed { term, positions -> - for (p in positions) { - candidatePos[n] = p - candidateTerm[n++] = term + private fun bestCluster( + occurrences: List, + visible: VisibleIndex, + lineStart: Int, + lineEnd: Int, + ): Pair? { + if (occurrences.isEmpty()) return null + val candidates = occurrences.take(MAX_CANDIDATES) + val lineHasMatch = candidates.any { it.start >= lineStart && it.end <= lineEnd } + val units = candidates.maxOf { it.unit } + 1 + val bestLevel = IntArray(units) + val reach = CLUSTER_REACH + + var best: Pair? = null + var bestScore = longArrayOf(-1, -1, -1, Long.MIN_VALUE) + for (i in candidates.indices) { + bestLevel.fill(EvidenceLevel.NONE) + var inLine = false + var j = i + while (j < candidates.size && visible.at(candidates[j].end) - visible.at(candidates[i].start) <= reach) { + val o = candidates[j] + if (o.level < bestLevel[o.unit]) bestLevel[o.unit] = o.level + if (o.start >= lineStart && o.end <= lineEnd) inLine = true + j++ } - } - // The same occurrences sorted by position, to count the terms around each one - val byPosition = (0 until count).sortedBy { candidatePos[it] } - val sortedPos = IntArray(count) { candidatePos[byPosition[it]] } - val sortedTerm = IntArray(count) { candidateTerm[byPosition[it]] } - val seen = IntArray(anchorTerms.size) - var stamp = 0 - - val maxPossibleScore = anchorTerms.size - var bestScore = 0 - for (c in 0 until count) { - val pos = candidatePos[c] - val length = anchorTerms[candidateTerm[c]].length - val windowStart = pos - effContext - val windowEnd = pos + length + effContext - stamp++ - var uniqueTerms = 0 - var i = lowerBound(sortedPos, windowStart) - while (i < count && sortedPos[i] <= windowEnd) { - val term = sortedTerm[i++] - if (seen[term] != stamp) { - seen[term] = stamp - uniqueTerms++ + if (j == i) j = i + 1 // a single match longer than the window + // The snippet starts at its group: in the hit line, so the result shows its own line first + if (lineHasMatch && !candidates[i].let { it.start >= lineStart && it.end <= lineEnd }) continue + var distinct = 0L + var sureness = 0L + for (level in bestLevel) { + if (level != EvidenceLevel.NONE) { + distinct++ + sureness += EvidenceLevel.NONE - level } } - val score = uniqueTerms * 100 + length - if (score > bestScore) { + val end = candidates.subList(i, j).maxOf { it.end } + val score = longArrayOf(if (inLine) 1 else 0, distinct, sureness, (j - i).toLong()) + if (compare(score, bestScore) > 0) { bestScore = score - plainIdx = pos - plainLen = length - if (uniqueTerms >= maxPossibleScore) break + best = candidates[i].start to end } } - return plainIdx to plainLen - } - - private fun lowerBound(sorted: IntArray, value: Int): Int { - var lo = 0 - var hi = sorted.size - while (lo < hi) { - val mid = (lo + hi) ushr 1 - if (sorted[mid] < value) lo = mid + 1 else hi = mid - } - return lo + return best } - private fun isWordBoundary(text: String, index: Int): Boolean { - if (index < 0 || index >= text.length) return true - val ch = text[index] - return ch.isWhitespace() || !ch.isLetterOrDigit() + private fun compare(a: LongArray, b: LongArray): Int { + for (k in a.indices) if (a[k] != b[k]) return a[k].compareTo(b[k]) + return 0 } - private fun mergeIntervals(ranges: List): List { - if (ranges.isEmpty()) return ranges - val out = mutableListOf() - var cur = ranges[0] - for (i in 1 until ranges.size) { - val r = ranges[i] - if (r.first <= cur.last + 1) { - cur = cur.first..maxOf(cur.last, r.last) - } else { - out += cur - cur = r - } + private fun withBold(raw: String, start: Int, end: Int, spans: List): String { + val sb = StringBuilder(end - start + spans.size * 7) + var at = start + for (span in spans) { + sb.append(raw, at, span.first).append("").append(raw, span.first, span.last + 1).append("") + at = span.last + 1 } - out += cur - return out + return sb.append(raw, at, end).toString() } - private fun insertBoldTags(text: String, intervals: List): String { - if (intervals.isEmpty()) return text - val sb = StringBuilder(text) - for (r in intervals.asReversed()) { - val start = r.first.coerceIn(0, sb.length) - val end = (r.last + 1).coerceIn(0, sb.length) - if (end > start) { - sb.insert(end, "") - sb.insert(start, "") - } - } - return sb.toString() + // The start of the first whole word from [from] (no later than [limit]) + private fun snapForward(raw: String, from: Int, limit: Int): Int { + if (raw[from - 1].isWhitespace()) return from + var i = from + while (i < limit && !raw[i].isWhitespace()) i++ + return if (i < limit) i + 1 else from } -} -/** Finds the occurrences of many terms in one pass over a text (Aho-Corasick automaton). */ -internal class AhoCorasick(terms: List) { - private val termLengths = IntArray(terms.size) { terms[it].length } - private val next = ArrayList>() - private val fail = ArrayList() - - // Terms ending at each node, its own then those of its suffix links - private val outputs = ArrayList() - - init { - next += HashMap() - fail += 0 - val own = ArrayList>().apply { add(mutableListOf()) } - terms.forEachIndexed { index, term -> - if (term.isEmpty()) return@forEachIndexed - var node = 0 - for (ch in term) { - node = next[node].getOrPut(ch) { - next += HashMap() - fail += 0 - own.add(mutableListOf()) - next.size - 1 - } - } - own[node] += index - } - // Breadth-first: suffix links, and outputs merged along them - repeat(next.size) { outputs += IntArray(0) } - outputs[0] = own[0].toIntArray() - val queue = ArrayDeque() - for (child in next[0].values) { - fail[child] = 0 - queue += child - } - while (queue.isNotEmpty()) { - val node = queue.removeFirst() - outputs[node] = (own[node] + outputs[fail[node]].toList()).toIntArray() - for ((ch, child) in next[node]) { - var f = fail[node] - while (f != 0 && ch !in next[f]) f = fail[f] - fail[child] = next[f][ch]?.takeIf { it != child } ?: 0 - queue += child - } - } + // The end of the last whole word before [from] (no earlier than [limit]) + private fun snapBackward(raw: String, from: Int, limit: Int): Int { + if (raw[from].isWhitespace()) return from + var i = from + while (i > limit && !raw[i - 1].isWhitespace()) i-- + return if (i > limit) i - 1 else from } - /** Per term, the start positions of its first [max] occurrences in [text], overlapping ones included. */ - fun firstOccurrences(text: String, max: Int): Array { - val found = Array(termLengths.size) { IntArray(max) } - val counts = IntArray(termLengths.size) - var node = 0 - for (i in text.indices) { - val ch = text[i] - while (node != 0 && ch !in next[node]) node = fail[node] - node = next[node][ch] ?: 0 - for (term in outputs[node]) { - // Matches of a term come by end, so by start: the first ones found are the first ones - if (counts[term] < max) found[term][counts[term]++] = i - termLengths[term] + 1 + /** Positions counted in visible characters: the diacritics take no room on screen. */ + private class VisibleIndex(raw: String) { + // Visible characters before each offset + private val before = IntArray(raw.length + 1) + + // Offset of each visible character + private val offsets: IntArray + val total: Int + val hasDiacritics: Boolean + + init { + var count = 0 + for (i in raw.indices) { + before[i] = count + if (!HebrewTextUtils.isNikudOrTeamim(raw[i])) count++ } + before[raw.length] = count + total = count + hasDiacritics = count != raw.length + offsets = IntArray(count) + for (i in raw.indices) if (!HebrewTextUtils.isNikudOrTeamim(raw[i])) offsets[before[i]] = i } - return Array(termLengths.size) { found[it].copyOf(counts[it]) } + + fun at(offset: Int): Int = before[offset.coerceIn(0, before.size - 1)] + + fun offsetOf(visibleIndex: Int): Int = if (visibleIndex >= total) before.size - 1 else offsets[visibleIndex] } } diff --git a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SnippetProvider.kt b/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SnippetProvider.kt index 02e312d6d..858dc31b0 100644 --- a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SnippetProvider.kt +++ b/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SnippetProvider.kt @@ -1,5 +1,8 @@ package io.github.kdroidfilter.seforimlibrary.search +import org.jsoup.Jsoup +import org.jsoup.safety.Safelist + /** * Metadata about a line needed to fetch its snippet source text. * @@ -14,32 +17,23 @@ data class LineSnippetInfo( ) /** - * Provider interface for fetching snippet source text from a data store. - * - * The search engine uses this to retrieve the full text content needed for - * snippet generation. Implementations typically fetch from a database and - * may include neighboring lines for better context. + * The text a hit's snippet is cut from: the hit line, HTML-cleaned (escaped), with its neighbors when it is short. * - * ## Implementation Notes - * - Should return HTML-cleaned text (no raw HTML tags) - * - May include neighboring lines for context (typically 4 lines before/after) - * - Should handle missing lines gracefully (omit from result map) - * - Should be efficient for batch lookups (single DB query for all lines) + * @property text the source text + * @property lineStart start of the hit line in [text] + * @property lineEnd end of the hit line in [text] + */ +data class SnippetSource( + val text: String, + val lineStart: Int, + val lineEnd: Int +) + +/** + * Provider interface for fetching snippet source text from a data store. * - * ## Example Implementation - * ```kotlin - * class RepositorySnippetProvider( - * private val repository: SeforimRepository - * ) : SnippetProvider { - * override fun getSnippetSources(lines: List): Map { - * return repository.getSnippetSourcesForLines( - * lines.map { it.lineId }, - * neighborWindow = 4, - * minLength = 280 - * ) - * } - * } - * ``` + * The search engine uses this to retrieve the full text content needed for snippet generation. Implementations + * typically load the lines from a database and build the sources with [SnippetSources.build]. * * @see SearchEngine for how this provider is used during search */ @@ -48,7 +42,72 @@ fun interface SnippetProvider { * Fetches snippet source text for multiple lines in a single batch. * * @param lines List of line metadata for which to fetch text - * @return Map of lineId to its source text. Missing lines should be omitted. + * @return Map of lineId to its source. Missing lines should be omitted. + */ + suspend fun getSnippetSources(lines: List): Map +} + +/** Builds snippet sources the same way for every data store. */ +object SnippetSources { + /** Lines kept on each side of a short hit line */ + const val NEIGHBOR_WINDOW = 4 + + /** A hit line this long (cleaned) is shown alone */ + const val MIN_LENGTH = 280 + + private val WHITESPACE = Regex("\\s+") + + /** + * Sources for [lines]: each hit line HTML-cleaned, with its [NEIGHBOR_WINDOW] neighbors on each side when shorter + * than [MIN_LENGTH]. Only the windows around the hits are loaded, through [loadLines] (a book's raw HTML lines + * from one index to another, both included, by index); only the lines used are cleaned. */ - suspend fun getSnippetSources(lines: List): Map + suspend fun build( + lines: List, + loadLines: suspend (bookId: Long, fromIndex: Int, toIndex: Int) -> Map, + ): Map { + val result = HashMap() + for ((bookId, bookLines) in lines.groupBy { it.bookId }) { + // Overlapping windows merged: a book's hits can be thousands of lines apart + val contentByIndex = HashMap() + for (range in windows(bookLines.map { it.lineIndex })) { + contentByIndex.putAll(loadLines(bookId, range.first, range.last)) + } + val plainByIndex = HashMap() + fun plain(index: Int): String? = + plainByIndex[index] ?: contentByIndex[index]?.let { clean(it).also { cleaned -> plainByIndex[index] = cleaned } } + + for (info in bookLines) { + val line = plain(info.lineIndex) ?: continue + if (line.length >= MIN_LENGTH) { + result[info.lineId] = SnippetSource(line, 0, line.length) + continue + } + val sb = StringBuilder() + var lineStart = 0 + for (index in (info.lineIndex - NEIGHBOR_WINDOW).coerceAtLeast(0)..info.lineIndex + NEIGHBOR_WINDOW) { + val text = plain(index) ?: continue + if (sb.isNotEmpty()) sb.append(' ') + if (index == info.lineIndex) lineStart = sb.length + sb.append(text) + } + result[info.lineId] = SnippetSource(sb.toString(), lineStart, lineStart + line.length) + } + } + return result + } + + /** [html] as snippet text: tags dropped, entities escaped, whitespace collapsed. */ + fun clean(html: String): String = Jsoup.clean(html, Safelist.none()).replace(WHITESPACE, " ").trim() + + private fun windows(lineIndexes: List): List { + val out = ArrayList() + for (index in lineIndexes.sorted()) { + val start = (index - NEIGHBOR_WINDOW).coerceAtLeast(0) + val end = index + NEIGHBOR_WINDOW + val last = out.lastOrNull() + if (last != null && start <= last.last + 1) out[out.lastIndex] = last.first..maxOf(last.last, end) else out += start..end + } + return out + } } diff --git a/search/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/MagicDictionaryIndexTest.kt b/search/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/MagicDictionaryIndexTest.kt index 973707aa1..900fef7d8 100644 --- a/search/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/MagicDictionaryIndexTest.kt +++ b/search/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/MagicDictionaryIndexTest.kt @@ -283,6 +283,29 @@ class MagicDictionaryIndexTest { } } + @Test + fun `expansion prefers an entry the token is a form of over a larger one reached by a variant`() { + val tempFile = createValidTestDatabase() + try { + DriverManager.getConnection("jdbc:sqlite:${tempFile.toAbsolutePath()}").use { conn -> + conn.createStatement().use { stmt -> + // לעולם is a form of עולם, and a variant of a form of the much larger יום + stmt.execute("INSERT INTO base (id, value) VALUES (1, 'עולם'), (2, 'יום')") + stmt.execute("INSERT INTO surface (id, value, base_id) VALUES (1, 'לעולם', 1), (2, 'עולם', 1)") + stmt.execute( + "INSERT INTO surface (id, value, base_id) VALUES (3, 'ימים', 2), (4, 'יומא', 2), (5, 'ביום', 2), (6, 'יום', 2)" + ) + stmt.execute("INSERT INTO variant (id, value) VALUES (1, 'לעולם')") + stmt.execute("INSERT INTO surface_variant (surface_id, variant_id) VALUES (3, 1)") + } + } + val index = assertNotNull(MagicDictionaryIndex.load(simpleNorm, tempFile)) + assertEquals(listOf("עולם"), assertNotNull(index.expansionFor("לעולם")).base) + } finally { + Files.deleteIfExists(tempFile) + } + } + // --- Caching tests --- @Test diff --git a/search/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/SnippetBuilderTest.kt b/search/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/SnippetBuilderTest.kt index b0c61e463..546e17890 100644 --- a/search/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/SnippetBuilderTest.kt +++ b/search/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/SnippetBuilderTest.kt @@ -1,213 +1,149 @@ package io.github.kdroidfilter.seforimlibrary.search -import kotlin.random.Random import kotlin.test.Test import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertTrue class SnippetBuilderTest { + private fun plan(vararg tokens: TokenForms, phrases: List = emptyList()) = HighlightPlan.build(tokens.toList(), phrases) + + private fun words(token: String) = TokenForms(HebrewTextUtils.normalizeHebrew(token)) + + private fun bold(snippet: String): List = Regex("(.*?)").findAll(snippet).map { it.groupValues[1] }.toList() + @Test - fun `matches the reference snippet builder on random texts`() { - val random = Random(42) - repeat(4000) { case -> - val text = randomText(random) - val words = text.split(' ').filter { it.isNotEmpty() } - val plainWords = words.map { HebrewTextUtils.replaceFinalsWithBase(HebrewTextUtils.stripDiacritics(it)) } - val highlight = List(random.nextInt(0, 40)) { randomTerm(random, plainWords) } - // Like buildAnchorTerms: distinct, non-blank, longest first - val anchors = (highlight + List(random.nextInt(0, 5)) { randomTerm(random, plainWords) }) - .map { it.trim() } - .filter { it.length >= 2 } - .distinct() - .sortedByDescending { it.length } - assertEquals( - referenceSnippet(text, anchors, highlight), - SnippetBuilder.build(text, SnippetTerms(anchors, highlight)), - "case $case: text=$text anchors=$anchors highlight=$highlight", - ) - } - } - - @Test - fun `aho corasick finds overlapping occurrences in order`() { - val matcher = AhoCorasick(listOf("אבא", "בא", "א", "")) - val found = matcher.firstOccurrences("אבאבא", max = 5) - assertEquals(listOf(0, 2), found[0].toList()) - assertEquals(listOf(1, 3), found[1].toList()) - assertEquals(listOf(0, 2, 4), found[2].toList()) - assertEquals(emptyList(), found[3].toList()) - assertEquals(listOf(0, 2), matcher.firstOccurrences("אבאבא", max = 2)[2].toList()) - } - - private val letters = "אבגדהוזחטיכלמנסעפצקרשתךםןףץ" - private val marks = listOf('\u05B0', '\u05B4', '\u05B7', '\u05BC', '\u0591', '\u05C1') - private val punctuation = listOf(",", ".", ":", "׳", "\"", "'", "(", ")", "-", "־") - - private fun randomWord(random: Random): String = buildString { - repeat(random.nextInt(1, 7)) { - append(letters[random.nextInt(letters.length)]) - if (random.nextInt(4) == 0) append(marks[random.nextInt(marks.size)]) - } - if (random.nextInt(25) == 0) append("AbC") - if (random.nextInt(8) == 0) append(punctuation[random.nextInt(punctuation.size)]) - } - - private fun randomText(random: Random): String { - // Short lines and long ones, so the window is sometimes the whole text, sometimes a slice - val words = List(random.nextInt(0, if (random.nextBoolean()) 30 else 400)) { randomWord(random) } - return words.joinToString(" ") - } - - private fun randomTerm(random: Random, plainWords: List): String { - val word = if (plainWords.isNotEmpty() && random.nextInt(4) != 0) plainWords[random.nextInt(plainWords.size)] else { - HebrewTextUtils.replaceFinalsWithBase(HebrewTextUtils.stripDiacritics(randomWord(random))) - } - return when (random.nextInt(8)) { - 0 -> word.take(4) - 1 -> word + "$" - 2 -> word + " " + (plainWords.randomOrNull(random) ?: "") - 3 -> word.lowercase().uppercase() - 4 -> word.drop(1) - else -> word - } + fun `highlights whole words, never part of one`() { + val snippet = SnippetBuilder.build("נבראו בראשית ברא אלהים", plan(words("ברא"))) + assertEquals(listOf("ברא"), bold(snippet)) } -} -// The snippet builder before SnippetBuilder, kept verbatim as the reference -private fun referenceSnippet(raw: String, anchorTerms: List, highlightTerms: List, context: Int = 220): String { - if (raw.isEmpty()) return "" - val (plain, mapToOrig) = HebrewTextUtils.stripDiacriticsWithMap(raw) - val hasDiacritics = plain.length != raw.length - val effContext = if (hasDiacritics) maxOf(context, 360) else context - val plainSearch = HebrewTextUtils.replaceFinalsWithBase(plain) - - // Find best anchor position: where most terms cluster together - // Optimized: limit occurrences per term, early exit on perfect score - var plainIdx = 0 - var plainLen = anchorTerms.firstOrNull()?.length ?: 0 - - if (anchorTerms.isNotEmpty()) { - val maxOccPerTerm = 5 // Limit occurrences per term for perf - val positions = mutableListOf>() // (position, term) - - for (term in anchorTerms) { - if (term.isEmpty()) continue - var from = 0 - var count = 0 - while (from <= plainSearch.length - term.length && count < maxOccPerTerm) { - val idx = plainSearch.indexOf(term, startIndex = from) - if (idx == -1) break - positions.add(idx to term) - from = idx + 1 - count++ - } - } - - if (positions.isNotEmpty()) { - val maxPossibleScore = anchorTerms.size - var bestScore = 0 - - for ((pos, term) in positions) { - // Count unique terms in window around this position - val windowStart = pos - effContext - val windowEnd = pos + term.length + effContext - var uniqueTerms = 0 - val seen = mutableSetOf() - for ((p, t) in positions) { - if (p in windowStart..windowEnd && seen.add(t)) uniqueTerms++ - } - val score = uniqueTerms * 100 + term.length - - if (score > bestScore) { - bestScore = score - plainIdx = pos - plainLen = term.length - // Early exit if we found all terms clustered - if (uniqueTerms >= maxPossibleScore) break - } - } - } - } - val plainStart = (plainIdx - effContext).coerceAtLeast(0) - val plainEnd = (plainIdx + plainLen + effContext).coerceAtMost(plain.length) - val origStart = HebrewTextUtils.mapToOrigIndex(mapToOrig, plainStart) - val origEnd = HebrewTextUtils.mapToOrigIndex(mapToOrig, plainEnd).coerceAtMost(raw.length) - - val base = raw.substring(origStart, origEnd) - val basePlain = plain.substring(plainStart, plainEnd) - val basePlainSearch = HebrewTextUtils.replaceFinalsWithBase(basePlain) - val baseMap = IntArray(plainEnd - plainStart) { idx -> - (mapToOrig[plainStart + idx] - origStart).coerceIn(0, base.length.coerceAtLeast(1) - 1) - } - - val pool = (highlightTerms + highlightTerms.map { it.trimEnd('$') }).distinct().filter { it.isNotBlank() } - val intervals = mutableListOf() - val basePlainLower = basePlainSearch.lowercase() - - fun isWordBoundary(text: String, index: Int): Boolean { - if (index < 0 || index >= text.length) return true - val ch = text[index] - return ch.isWhitespace() || !ch.isLetterOrDigit() - } - - for (term in pool) { - if (term.isEmpty()) continue - val t = term.lowercase() - var from = 0 - while (from <= basePlainLower.length - t.length && t.isNotEmpty()) { - val idx = basePlainLower.indexOf(t, startIndex = from) - if (idx == -1) break - - val isAtWordStart = isWordBoundary(basePlainLower, idx - 1) - val isAtWordEnd = isWordBoundary(basePlainLower, idx + t.length) - val isWholeWord = isAtWordStart && isAtWordEnd - val shouldHighlight = isWholeWord - - if (shouldHighlight) { - val startOrig = HebrewTextUtils.mapToOrigIndex(baseMap, idx) - val endOrig = HebrewTextUtils.mapToOrigIndex(baseMap, (idx + t.length - 1)) + 1 - if (startOrig in 0 until endOrig && endOrig <= base.length) { - intervals += (startOrig until endOrig) - } - } - from = idx + 1 - } - } - - val merged = mergeIntervals(intervals.sortedBy { it.first }) - val highlighted = insertBoldTags(base, merged) - val prefix = if (origStart > 0) "..." else "" - val suffix = if (origEnd < raw.length) "..." else "" - return prefix + highlighted + suffix -} + @Test + fun `a word behind prefixes or inside a longer word counts from four letters`() { + assertEquals(listOf("ובבראשית", "בראשיתו"), bold(SnippetBuilder.build("ובבראשית ואחר בראשיתו", plan(words("בראשית"))))) + // Three letters: prefixes only + assertEquals(listOf("וברא"), bold(SnippetBuilder.build("וברא נבראו", plan(words("ברא"))))) + } -private fun mergeIntervals(ranges: List): List { - if (ranges.isEmpty()) return ranges - val out = mutableListOf() - var cur = ranges[0] - for (i in 1 until ranges.size) { - val r = ranges[i] - if (r.first <= cur.last + 1) { - cur = cur.first .. maxOf(cur.last, r.last) - } else { - out += cur - cur = r - } - } - out += cur - return out -} + @Test + fun `a two-letter word only takes vav or he as prefix`() { + assertEquals(listOf("כל וכל הכל"), bold(SnippetBuilder.build("כל וכל הכל שכל לכל", plan(words("כל"))))) + } -private fun insertBoldTags(text: String, intervals: List): String { - if (intervals.isEmpty()) return text - val sb = StringBuilder(text) - for (r in intervals.asReversed()) { - val start = r.first.coerceIn(0, sb.length) - val end = (r.last + 1).coerceIn(0, sb.length) - if (end > start) { - sb.insert(end, "") - sb.insert(start, "") - } - } - return sb.toString() -} + @Test + fun `the bold covers the word's diacritics and final letters match`() { + val snippet = SnippetBuilder.build("תִּשְׁבְּתוּ וְהָעוֹלָם", plan(words("עולם"), words("תשבתו"))) + assertEquals(listOf("תִּשְׁבְּתוּ וְהָעוֹלָם"), bold(snippet)) + } + + @Test + fun `acronyms match with any quote marks`() { + val query = plan(words("רמב\"ם")) + assertEquals(listOf("הרמב״ם", "רמב\"ם"), bold(SnippetBuilder.build("כתב הרמב״ם וכן רמב\"ם", query))) + assertEquals(listOf("רש״י"), bold(SnippetBuilder.build("פירש רש״י", plan(words("רש\"י"))))) + } + + @Test + fun `the divine name's lone he needs its geresh`() { + val hashem = TokenForms("ה", extraLiterals = listOf("ה'", "יהוה")) + val snippet = SnippetBuilder.build("סימן ה אמר ה׳ אלקים ויאמר ה' ליהוה", plan(hashem, words("אלקים"))) + assertEquals(listOf("ה׳ אלקים", "ה' ליהוה"), bold(snippet)) + } + + @Test + fun `inflections count, not the dictionary's other forms`() { + val shabbat = TokenForms("שבת", inflections = listOf("שבתות", "שבת")) + assertEquals(listOf("שבת ושבתות"), bold(SnippetBuilder.build("ביום השביעי שבת ושבתות", plan(shabbat)))) + } + + @Test + fun `a quoted phrase shows as its words in a row only`() { + val snippet = SnippetBuilder.build("אין שם אין עומדין להתפלל אלא", plan(phrases = listOf("אין עומדין להתפלל"))) + assertEquals(listOf("אין עומדין להתפלל"), bold(snippet)) + } + + @Test + fun `dictionary forms of several words`() { + val token = TokenForms("כלבו", inflections = listOf("כל בו")) + assertEquals(listOf("כל בו"), bold(SnippetBuilder.build("כתב בעל כל בו כל הלכה", plan(token)))) + } + @Test + fun `matched words in a row read as one span, across a maqaf too`() { + assertEquals(listOf("קריעת ים־סוף"), bold(SnippetBuilder.build("נס קריעת ים־סוף גדול", plan(words("קריעת"), words("ים"), words("סוף"))))) + assertEquals(listOf("ים", "סוף"), bold(SnippetBuilder.build("ים, סוף", plan(words("ים"), words("סוף"))))) + } + + @Test + fun `html entities are not words`() { + val snippet = SnippetBuilder.build("a & amp b", plan(TokenForms("amp"))) + assertEquals("a & amp b", snippet) + } + + @Test + fun `the window shows the hit line's match over richer neighbors`() { + val neighbor = (1..60).joinToString(" ") { "אלף בית גימל" } + val line = "כאן אלף לבדו" + val source = "$neighbor $line" + val snippet = SnippetBuilder.build(source, plan(words("אלף"), words("בית"), words("גימל")), neighbor.length + 1, source.length) + assertTrue(snippet.contains("כאן אלף לבדו"), snippet) + } + + @Test + fun `the snippet starts in the hit line even when the line before has more of the query`() { + val before = "אלף בית גימל" + val line = "ושוב אלף כאן" + val source = "$before $line" + val snippet = SnippetBuilder.build(source, plan(words("אלף"), words("בית"), words("גימל")), before.length + 1, source.length) + assertTrue(snippet.substringBefore("").contains("ושוב") || snippet.startsWith("...ושוב"), snippet) + } + + @Test + fun `the window gathers the most query words and cuts between words`() { + val filler = (1..80).joinToString(" ") { "מילה$it" } + val text = "ראשון $filler שני שלישי $filler" + val snippet = SnippetBuilder.build(text, plan(words("ראשון"), words("שני"), words("שלישי"))) + assertTrue("שני שלישי" in snippet, snippet) + // Whole words at the edges + val inner = snippet.removePrefix("...").removeSuffix("...") + val edges = inner.replace(Regex(""), "").split(' ') + assertTrue(edges.first().startsWith("מילה") && edges.last().startsWith("מילה"), snippet) + assertTrue(edges.all { it.matches(Regex("מילה\\d+|שני|שלישי")) }, snippet) + } + + @Test + fun `no match keeps the line start`() { + val before = (1..80).joinToString(" ") { "לפני$it" } + val source = "$before שורה כאן" + val snippet = SnippetBuilder.build(source, plan(words("אחר")), before.length + 1, source.length) + assertFalse("" in snippet) + assertEquals("...שורה כאן", snippet) + } + + @Test + fun `the snippet starts just before the matches, at their sentence when close`() { + val before = (1..60).joinToString(" ") { "מילה$it" } + val snippet = SnippetBuilder.build("$before. משפט חדש כאן מטרה ועוד", plan(words("מטרה"))) + assertTrue(snippet.startsWith("...משפט חדש כאן מטרה"), snippet) + // No sentence end near: a few words of context + val far = SnippetBuilder.build("$before מטרה ועוד", plan(words("מטרה"))) + val lead = far.removePrefix("...").substringBefore("") + assertTrue(lead.length in 1..40 && lead.startsWith("מילה"), far) + } + + @Test + fun `a passage snippet bolds the passage as one span`() { + val text = "פתיחה ארוכה. ואמר החכם דבר נאה מאוד לתלמידיו וסיים" + val passage = "דבר נאה מאוד" + val at = text.indexOf(passage) + val snippet = SnippetBuilder.build(text, listOf(Occurrence(at, at + passage.length, 0, EvidenceLevel.LITERAL))) + assertEquals(listOf(passage), bold(snippet)) + } + + @Test + fun `highlight ranges of a plain text are the snippet's words`() { + val text = "וּבְרֵאשִׁית ברא נבראו" + val ranges = mergedRanges(text, plan(words("בראשית"), words("ברא")).highlights(text)) + assertEquals(listOf("וּבְרֵאשִׁית ברא"), ranges.map { text.substring(it.first, it.last + 1) }) + } +}