Skip to content

Commit c9c893b

Browse files
committed
Merge pull request #88 from kdroidFilter/feat/semantic-highlight
feat(search): embedding-only semantic highlight + native-image dense fix
2 parents 020d17b + 6784374 commit c9c893b

4 files changed

Lines changed: 185 additions & 71 deletions

File tree

‎search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/HybridSearchEngine.kt‎

Lines changed: 152 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ import kotlinx.coroutines.Dispatchers
55
import kotlinx.coroutines.sync.Mutex
66
import kotlinx.coroutines.sync.withLock
77
import kotlinx.coroutines.withContext
8+
import org.jsoup.Jsoup
9+
import org.jsoup.safety.Safelist
810
import java.nio.file.Files
911
import java.nio.file.Path
1012

@@ -47,7 +49,7 @@ class HybridSearchEngine(
4749
// Cheap, no-load check: are the model + vector index even present? Decides whether to
4850
// take the dense path (then load lazily); the actual OrtSession is built in fuse().
4951
private val denseConfigured: Boolean =
50-
indexDir != null && Files.isDirectory(indexDir) && SeforimEmbedder.isAvailable(modelDir)
52+
Files.isDirectory(indexDir) && SeforimEmbedder.isAvailable(modelDir)
5153

5254
val denseEnabled: Boolean get() = embedder != null && vectorSearcher != null
5355

@@ -58,7 +60,7 @@ class HybridSearchEngine(
5860
if (denseTried) return
5961
withContext(Dispatchers.IO) {
6062
val emb = SeforimEmbedder.tryLoad(modelDir)
61-
val vs = if (emb != null && indexDir != null && Files.isDirectory(indexDir)) {
63+
val vs = if (emb != null && Files.isDirectory(indexDir)) {
6264
runCatching { VectorSearcher(indexDir) }.getOrNull()
6365
} else null
6466
if (vs != null) {
@@ -100,8 +102,46 @@ class HybridSearchEngine(
100102
override fun buildSnippet(rawText: String, query: String, near: Int): String =
101103
lexical.buildSnippet(rawText, query, near)
102104

103-
override fun buildHighlightTerms(query: String): List<String> =
104-
lexical.buildHighlightTerms(query)
105+
/**
106+
* Picks the passage of [text] closest in meaning to [query], using the SAME dense
107+
* encoder as semantic search — so the highlight reflects meaning instead of scattering
108+
* dictionary word matches. Returns the winning passage verbatim, or null when dense is
109+
* unavailable or the text isn't worth localizing (a single short clause).
110+
*/
111+
override suspend fun semanticSpan(query: String, text: String): String? {
112+
if (query.isBlank() || text.isBlank()) return null
113+
ensureDense()
114+
val emb = embedder ?: return null
115+
val clauses = splitClauses(text)
116+
if (clauses.size < 2) return null
117+
return withContext(Dispatchers.Default) {
118+
val qVec = emb.embed(query)
119+
clauses
120+
.map { it to cosine(qVec, emb.embed(it)) }
121+
.maxByOrNull { it.second }
122+
?.first
123+
}
124+
}
125+
126+
/**
127+
* Embedding-based find-in-page in one book: dense KNN over the index scoped to [bookId].
128+
* Reuses the line vectors already in the index, so this is just the query embedding + a
129+
* filtered KNN — the per-line passage is computed by the caller on the displayed text.
130+
*/
131+
override suspend fun denseReady(): Boolean {
132+
ensureDense()
133+
return denseEnabled
134+
}
135+
136+
override suspend fun semanticFind(query: String, bookId: Long, limit: Int): List<Long> {
137+
if (query.isBlank()) return emptyList()
138+
ensureDense()
139+
val emb = embedder ?: return emptyList()
140+
val vs = vectorSearcher ?: return emptyList()
141+
return withContext(Dispatchers.Default) {
142+
vs.search(emb.embed(query), limit, baseBookOnly = false, bookIds = listOf(bookId)).map { it.lineId }
143+
}
144+
}
105145

106146
override fun computeFacets(
107147
query: String, near: Int, bookFilter: Long?, categoryFilter: Long?,
@@ -114,20 +154,31 @@ class HybridSearchEngine(
114154
runCatching { lexical.close() }
115155
}
116156

157+
/** The fused, RRF-ordered hits plus the ids that the lexical path matched (so the dense-only
158+
* ones can get a meaning-based snippet lazily — their lexical snippet is meaningless). */
159+
private class FusedResult(val hits: List<LineHit>, val lexicalIds: Set<Long>)
160+
117161
/** Lexical page + dense KNN, fused by RRF, resolved to full LineHits. */
118-
private suspend fun fuse(query: String, near: Int, bookIds: Collection<Long>?, baseOnly: Boolean): List<LineHit> {
162+
private suspend fun fuse(query: String, near: Int, bookIds: Collection<Long>?, baseOnly: Boolean): FusedResult {
119163
ensureDense() // first call loads the model off-main (covered by the search spinner)
120164
val lexHits = lexical.openSession(query, near = near, bookIds = bookIds, baseBookOnly = baseOnly)
121165
?.use { it.nextPage(candidates)?.hits ?: emptyList() } ?: emptyList()
122166

123167
// Dense failed to load -> lexical-only result (still correct, just no semantic recall).
124168
val emb = embedder
125169
val vs = vectorSearcher
126-
if (emb == null || vs == null) return lexHits
170+
if (emb == null || vs == null) return FusedResult(lexHits, lexHits.map { it.lineId }.toSet())
127171

128-
val denseHits = withContext(Dispatchers.Default) {
129-
val qVec = emb.embed(query)
130-
vs.search(qVec, candidates, baseOnly, bookIds)
172+
// Don't let a dense failure sink the whole search: degrade to lexical and log. This
173+
// also catches native-image gaps (e.g. a missing Lucene KNN vectors-format SPI entry).
174+
val denseHits = try {
175+
withContext(Dispatchers.Default) {
176+
val qVec = emb.embed(query)
177+
vs.search(qVec, candidates, baseOnly, bookIds)
178+
}
179+
} catch (e: Exception) {
180+
logger.w(e) { "dense KNN failed; falling back to lexical-only" }
181+
return FusedResult(lexHits, lexHits.map { it.lineId }.toSet())
131182
}
132183

133184
val rrf = HashMap<Long, Double>()
@@ -145,7 +196,31 @@ class HybridSearchEngine(
145196
val hit = lexById[lineId] ?: resolveLine(lineId, bookOf[lineId] ?: -1L, query)
146197
if (hit != null) out += hit.copy(score = score.toFloat())
147198
}
148-
return out
199+
return FusedResult(out, lexById.keys)
200+
}
201+
202+
/**
203+
* Snippet for a dense-only hit: the lexical builder can't anchor (the query words aren't in
204+
* the text) and, fed the passage, it tokenizes and bolds each word separately — so short
205+
* function words (ואם, או…) get highlighted on their own. Instead, locate the passage closest
206+
* in meaning and bold it as ONE contiguous span inside a context window. Null -> keep lexical.
207+
*/
208+
private suspend fun semanticSnippet(query: String, rawText: String, near: Int): String? {
209+
// Jsoup.clean returns HTML-escaped text; substrings stay escaped, so we splice <b> in
210+
// directly (same convention as the lexical snippet builder).
211+
val clean = Jsoup.clean(rawText, Safelist.none())
212+
val passage = semanticSpan(query, clean) ?: return null
213+
val idx = clean.indexOf(passage)
214+
if (idx < 0) return lexical.buildSnippet(rawText, passage, near)
215+
val from = (idx - SNIPPET_CONTEXT).coerceAtLeast(0)
216+
val to = (idx + passage.length + SNIPPET_CONTEXT).coerceAtMost(clean.length)
217+
return buildString {
218+
if (from > 0) append("…")
219+
append(clean, from, idx)
220+
append("<b>").append(passage).append("</b>")
221+
append(clean, idx + passage.length, to)
222+
if (to < clean.length) append("…")
223+
}
149224
}
150225

151226
private inner class HybridSession(
@@ -154,24 +229,86 @@ class HybridSearchEngine(
154229
private val bookIds: Collection<Long>?,
155230
private val baseOnly: Boolean,
156231
) : SearchSession {
157-
private var fused: List<LineHit>? = null
232+
private var fused: FusedResult? = null
158233
private var offset = 0
159234

160235
override suspend fun nextPage(limit: Int): SearchPage? {
161236
val all = fused ?: fuse(query, near, bookIds, baseOnly).also { fused = it }
162-
if (offset >= all.size) return null
163-
val end = minOf(offset + limit, all.size)
164-
val slice = all.subList(offset, end)
237+
if (offset >= all.hits.size) return null
238+
val end = minOf(offset + limit, all.hits.size)
239+
// Re-snippet only the dense-only hits on THIS page (bounded work) so semantic
240+
// results show the passage that actually matched, not the line's opening words.
241+
val slice = all.hits.subList(offset, end).map { hit ->
242+
if (hit.lineId in all.lexicalIds) {
243+
hit
244+
} else {
245+
semanticSnippet(query, hit.rawText, near)?.let { hit.copy(snippet = it) } ?: hit
246+
}
247+
}
165248
offset = end
166-
return SearchPage(hits = slice, totalHits = all.size.toLong(), isLastPage = offset >= all.size)
249+
return SearchPage(hits = slice, totalHits = all.hits.size.toLong(), isLastPage = offset >= all.hits.size)
167250
}
168251

169252
override fun close() {}
170253
}
171254

255+
/**
256+
* Splits [text] into passage candidates: first on strong clause delimiters (incl. the
257+
* Hebrew sof-pasuk ׃), then sliding word windows over any segment too long to localize.
258+
* Substrings are verbatim slices of [text] (original spacing preserved) so the caller
259+
* can match them back diacritic-insensitively. Tiny fragments are dropped to keep a
260+
* highlight that is a *passage*, not a stray word.
261+
*/
262+
private fun splitClauses(text: String): List<String> {
263+
val delimiters = charArrayOf('.', '!', '?', ':', ';', '׃', '\n')
264+
val segments = ArrayList<IntRange>()
265+
var start = 0
266+
for (i in text.indices) {
267+
if (text[i] in delimiters) {
268+
if (i > start) segments += start..i
269+
start = i + 1
270+
}
271+
}
272+
if (start < text.length) segments += start until text.length
273+
274+
val out = ArrayList<String>()
275+
for (seg in segments) {
276+
val sub = text.substring(seg.first, seg.last + 1)
277+
val wordRanges = WORD_RE.findAll(sub).map { it.range }.toList()
278+
if (wordRanges.size < MIN_PASSAGE_WORDS) continue
279+
if (wordRanges.size <= MAX_PASSAGE_WORDS) {
280+
out += sub.substring(wordRanges.first().first, wordRanges.last().last + 1)
281+
} else {
282+
// Segment too long to localize: slide word windows over the original slice.
283+
var w = 0
284+
while (w < wordRanges.size) {
285+
val from = wordRanges[w].first
286+
val toIdx = minOf(w + MAX_PASSAGE_WORDS - 1, wordRanges.size - 1)
287+
out += sub.substring(from, wordRanges[toIdx].last + 1)
288+
if (toIdx == wordRanges.size - 1) break
289+
w += WINDOW_STRIDE_WORDS
290+
}
291+
}
292+
}
293+
return if (out.isEmpty()) listOf(text.trim()) else out
294+
}
295+
296+
private fun cosine(a: FloatArray, b: FloatArray): Float {
297+
var s = 0f
298+
val n = minOf(a.size, b.size)
299+
for (i in 0 until n) s += a[i] * b[i]
300+
return s
301+
}
302+
172303
companion object {
173304
private val logger = Logger.withTag("HybridSearch")
174305

306+
private val WORD_RE = Regex("\\S+")
307+
private const val MIN_PASSAGE_WORDS = 3
308+
private const val MAX_PASSAGE_WORDS = 14
309+
private const val WINDOW_STRIDE_WORDS = 8
310+
private const val SNIPPET_CONTEXT = 90 // chars of context kept on each side of the passage
311+
175312
fun create(
176313
lexical: LuceneSearchEngine,
177314
indexDir: Path,

‎search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/LuceneSearchEngine.kt‎

Lines changed: 0 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -218,53 +218,6 @@ class LuceneSearchEngine(
218218
return buildSnippetInternal(rawClean, anchorTerms, highlightTerms)
219219
}
220220

221-
override fun buildHighlightTerms(query: String): List<String> {
222-
val parsed = SearchQueryParser.parse(query)
223-
val norm = HebrewTextUtils.normalizeHebrew(parsed.freeText)
224-
// Verbatim tokens from the quoted phrases, highlighted without dictionary expansion.
225-
val exactPhraseTokens = parsed.exactPhrases
226-
.flatMap { analyzeToTerms(stdAnalyzer, HebrewTextUtils.normalizeHebrew(it)) ?: emptyList() }
227-
228-
if (norm.isBlank()) return filterTermsForHighlight(exactPhraseTokens)
229-
230-
val analyzedRaw = analyzeToTerms(stdAnalyzer, norm) ?: emptyList()
231-
val hasHashem = query.contains("ה׳") || query.contains("ה'")
232-
233-
// Filter single letters and stop words (same logic as buildSearchContext)
234-
val analyzedStd = analyzedRaw.filter { token ->
235-
if (token == "ה" && hasHashem) return@filter true
236-
if (token.any { it.isDigit() }) return@filter true
237-
token.length >= 2 && token !in setOf(
238-
"א", "ב", "ג", "ד", "ה", "ו", "ז", "ח", "ט", "י", "כ", "ל", "מ",
239-
"נ", "ס", "ע", "פ", "צ", "ק", "ר", "ש", "ת",
240-
)
241-
}
242-
243-
// Get dictionary expansions
244-
val tokenExpansions: Map<String, List<MagicDictionaryIndex.Expansion>> =
245-
analyzedStd.associateWith { token ->
246-
val expansion = magicDict?.expansionFor(token) ?: return@associateWith emptyList()
247-
listOf(expansion)
248-
}
249-
250-
// Filter hallucinations for highlighting
251-
val tokenExpansionsForHighlight = tokenExpansions.mapValues { (token, exps) ->
252-
exps.filter { exp -> !isHallucinatedExpansion(token, exp) }
253-
}
254-
255-
// Build expanded terms (filter 2-letter from expansions only)
256-
val allExpansionsForHighlight = tokenExpansionsForHighlight.values.flatten()
257-
val expandedTerms = allExpansionsForHighlight
258-
.flatMap { it.surface + it.variants + it.base }
259-
.filter { it.length > 2 }
260-
.distinct()
261-
262-
val ngramTerms = buildNgramTerms(analyzedStd, gram = 4)
263-
val hashemTerms = if (hasHashem) loadHashemHighlightTerms() else emptyList()
264-
265-
return filterTermsForHighlight(analyzedStd + expandedTerms + ngramTerms + hashemTerms + exactPhraseTokens)
266-
}
267-
268221
override fun close() {
269222
// Directory is closed automatically when readers are closed
270223
}

‎search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SearchEngine.kt‎

Lines changed: 24 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -86,17 +86,33 @@ interface SearchEngine : Closeable {
8686
fun buildSnippet(rawText: String, query: String, near: Int): String
8787

8888
/**
89-
* Builds a list of terms to highlight for a given query, using dictionary expansion.
89+
* Returns the contiguous passage within [text] whose meaning is closest to [query],
90+
* for semantic highlighting (using the same dense encoder as semantic search).
9091
*
91-
* This is useful for intelligent find-in-page that matches the same words
92-
* as the global search (including synonyms and morphological variants).
93-
* The terms are filtered to exclude hallucinated mappings and short words
94-
* that only came from dictionary expansion.
92+
* The result is a verbatim substring of [text] so callers can locate it with the
93+
* usual diacritic-insensitive matching and highlight that single span — instead of
94+
* scattering dictionary-expanded word matches that don't reflect meaning.
9595
*
96-
* @param query The search query in Hebrew
97-
* @return List of normalized terms to highlight (includes original tokens + expansions)
96+
* @return the best-matching passage, or null when dense search is unavailable or no
97+
* passage stands out (e.g. the text is a single short clause).
9898
*/
99-
fun buildHighlightTerms(query: String): List<String>
99+
suspend fun semanticSpan(query: String, text: String): String? = null
100+
101+
/**
102+
* Embedding-based find-in-page within a single book: returns the ids of the lines
103+
* semantically closest to [query] (dense KNN over the index, scoped to [bookId]),
104+
* ordered by relevance. Used by the "smart" find mode — the simple mode matches literal
105+
* words instead. The per-line passage to highlight is computed by the caller via
106+
* [semanticSpan] on the displayed text. Empty when dense search is unavailable.
107+
*/
108+
suspend fun semanticFind(query: String, bookId: Long, limit: Int): List<Long> = emptyList()
109+
110+
/**
111+
* Ensures the dense backend (embedding model + vector index) is loaded and reports whether
112+
* it is actually available. Useful for diagnostics and to decide whether semantic features
113+
* can run. Returns false for engines without a dense path.
114+
*/
115+
suspend fun denseReady(): Boolean = false
100116

101117
/**
102118
* Computes aggregate facet counts without loading full results.

‎search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/VectorSearcher.kt‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import org.apache.lucene.search.IndexSearcher
88
import org.apache.lucene.search.KnnFloatVectorQuery
99
import org.apache.lucene.search.Query
1010
import org.apache.lucene.store.FSDirectory
11+
import org.apache.lucene.store.NIOFSDirectory
1112
import java.io.Closeable
1213
import java.nio.file.Path
1314

@@ -23,7 +24,14 @@ data class DenseHit(val lineId: Long, val bookId: Long, val score: Float)
2324
* Returns line ids that are joined back to the DB by the caller.
2425
*/
2526
class VectorSearcher(indexDir: Path) : Closeable {
26-
private val dir = FSDirectory.open(indexDir)
27+
// GraalVM native image can't instantiate MMapDirectory's MemorySegmentIndexInputProvider
28+
// (Panama foreign downcalls) — use NIOFSDirectory there, like LuceneSearchEngine does.
29+
private val dir =
30+
if (System.getProperty("org.graalvm.nativeimage.imagecode") != null) {
31+
NIOFSDirectory(indexDir)
32+
} else {
33+
FSDirectory.open(indexDir)
34+
}
2735

2836
private fun filterQuery(baseBookOnly: Boolean, bookIds: Collection<Long>?): Query? {
2937
val b = BooleanQuery.Builder()

0 commit comments

Comments
 (0)