@@ -5,6 +5,8 @@ import kotlinx.coroutines.Dispatchers
55import kotlinx.coroutines.sync.Mutex
66import kotlinx.coroutines.sync.withLock
77import kotlinx.coroutines.withContext
8+ import org.jsoup.Jsoup
9+ import org.jsoup.safety.Safelist
810import java.nio.file.Files
911import java.nio.file.Path
1012
@@ -47,7 +49,7 @@ class HybridSearchEngine(
4749 // Cheap, no-load check: are the model + vector index even present? Decides whether to
4850 // take the dense path (then load lazily); the actual OrtSession is built in fuse().
4951 private val denseConfigured: Boolean =
50- indexDir != null && Files .isDirectory(indexDir) && SeforimEmbedder .isAvailable(modelDir)
52+ Files .isDirectory(indexDir) && SeforimEmbedder .isAvailable(modelDir)
5153
5254 val denseEnabled: Boolean get() = embedder != null && vectorSearcher != null
5355
@@ -58,7 +60,7 @@ class HybridSearchEngine(
5860 if (denseTried) return
5961 withContext(Dispatchers .IO ) {
6062 val emb = SeforimEmbedder .tryLoad(modelDir)
61- val vs = if (emb != null && indexDir != null && Files .isDirectory(indexDir)) {
63+ val vs = if (emb != null && Files .isDirectory(indexDir)) {
6264 runCatching { VectorSearcher (indexDir) }.getOrNull()
6365 } else null
6466 if (vs != null ) {
@@ -100,8 +102,46 @@ class HybridSearchEngine(
100102 override fun buildSnippet (rawText : String , query : String , near : Int ): String =
101103 lexical.buildSnippet(rawText, query, near)
102104
103- override fun buildHighlightTerms (query : String ): List <String > =
104- lexical.buildHighlightTerms(query)
105+ /* *
106+ * Picks the passage of [text] closest in meaning to [query], using the SAME dense
107+ * encoder as semantic search — so the highlight reflects meaning instead of scattering
108+ * dictionary word matches. Returns the winning passage verbatim, or null when dense is
109+ * unavailable or the text isn't worth localizing (a single short clause).
110+ */
111+ override suspend fun semanticSpan (query : String , text : String ): String? {
112+ if (query.isBlank() || text.isBlank()) return null
113+ ensureDense()
114+ val emb = embedder ? : return null
115+ val clauses = splitClauses(text)
116+ if (clauses.size < 2 ) return null
117+ return withContext(Dispatchers .Default ) {
118+ val qVec = emb.embed(query)
119+ clauses
120+ .map { it to cosine(qVec, emb.embed(it)) }
121+ .maxByOrNull { it.second }
122+ ?.first
123+ }
124+ }
125+
126+ /* *
127+ * Embedding-based find-in-page in one book: dense KNN over the index scoped to [bookId].
128+ * Reuses the line vectors already in the index, so this is just the query embedding + a
129+ * filtered KNN — the per-line passage is computed by the caller on the displayed text.
130+ */
131+ override suspend fun denseReady (): Boolean {
132+ ensureDense()
133+ return denseEnabled
134+ }
135+
136+ override suspend fun semanticFind (query : String , bookId : Long , limit : Int ): List <Long > {
137+ if (query.isBlank()) return emptyList()
138+ ensureDense()
139+ val emb = embedder ? : return emptyList()
140+ val vs = vectorSearcher ? : return emptyList()
141+ return withContext(Dispatchers .Default ) {
142+ vs.search(emb.embed(query), limit, baseBookOnly = false , bookIds = listOf (bookId)).map { it.lineId }
143+ }
144+ }
105145
106146 override fun computeFacets (
107147 query : String , near : Int , bookFilter : Long? , categoryFilter : Long? ,
@@ -114,20 +154,31 @@ class HybridSearchEngine(
114154 runCatching { lexical.close() }
115155 }
116156
157+ /* * The fused, RRF-ordered hits plus the ids that the lexical path matched (so the dense-only
158+ * ones can get a meaning-based snippet lazily — their lexical snippet is meaningless). */
159+ private class FusedResult (val hits : List <LineHit >, val lexicalIds : Set <Long >)
160+
117161 /* * Lexical page + dense KNN, fused by RRF, resolved to full LineHits. */
118- private suspend fun fuse (query : String , near : Int , bookIds : Collection <Long >? , baseOnly : Boolean ): List < LineHit > {
162+ private suspend fun fuse (query : String , near : Int , bookIds : Collection <Long >? , baseOnly : Boolean ): FusedResult {
119163 ensureDense() // first call loads the model off-main (covered by the search spinner)
120164 val lexHits = lexical.openSession(query, near = near, bookIds = bookIds, baseBookOnly = baseOnly)
121165 ?.use { it.nextPage(candidates)?.hits ? : emptyList() } ? : emptyList()
122166
123167 // Dense failed to load -> lexical-only result (still correct, just no semantic recall).
124168 val emb = embedder
125169 val vs = vectorSearcher
126- if (emb == null || vs == null ) return lexHits
170+ if (emb == null || vs == null ) return FusedResult ( lexHits, lexHits.map { it.lineId }.toSet())
127171
128- val denseHits = withContext(Dispatchers .Default ) {
129- val qVec = emb.embed(query)
130- vs.search(qVec, candidates, baseOnly, bookIds)
172+ // Don't let a dense failure sink the whole search: degrade to lexical and log. This
173+ // also catches native-image gaps (e.g. a missing Lucene KNN vectors-format SPI entry).
174+ val denseHits = try {
175+ withContext(Dispatchers .Default ) {
176+ val qVec = emb.embed(query)
177+ vs.search(qVec, candidates, baseOnly, bookIds)
178+ }
179+ } catch (e: Exception ) {
180+ logger.w(e) { " dense KNN failed; falling back to lexical-only" }
181+ return FusedResult (lexHits, lexHits.map { it.lineId }.toSet())
131182 }
132183
133184 val rrf = HashMap <Long , Double >()
@@ -145,7 +196,31 @@ class HybridSearchEngine(
145196 val hit = lexById[lineId] ? : resolveLine(lineId, bookOf[lineId] ? : - 1L , query)
146197 if (hit != null ) out + = hit.copy(score = score.toFloat())
147198 }
148- return out
199+ return FusedResult (out , lexById.keys)
200+ }
201+
202+ /* *
203+ * Snippet for a dense-only hit: the lexical builder can't anchor (the query words aren't in
204+ * the text) and, fed the passage, it tokenizes and bolds each word separately — so short
205+ * function words (ואם, או…) get highlighted on their own. Instead, locate the passage closest
206+ * in meaning and bold it as ONE contiguous span inside a context window. Null -> keep lexical.
207+ */
208+ private suspend fun semanticSnippet (query : String , rawText : String , near : Int ): String? {
209+ // Jsoup.clean returns HTML-escaped text; substrings stay escaped, so we splice <b> in
210+ // directly (same convention as the lexical snippet builder).
211+ val clean = Jsoup .clean(rawText, Safelist .none())
212+ val passage = semanticSpan(query, clean) ? : return null
213+ val idx = clean.indexOf(passage)
214+ if (idx < 0 ) return lexical.buildSnippet(rawText, passage, near)
215+ val from = (idx - SNIPPET_CONTEXT ).coerceAtLeast(0 )
216+ val to = (idx + passage.length + SNIPPET_CONTEXT ).coerceAtMost(clean.length)
217+ return buildString {
218+ if (from > 0 ) append(" …" )
219+ append(clean, from, idx)
220+ append(" <b>" ).append(passage).append(" </b>" )
221+ append(clean, idx + passage.length, to)
222+ if (to < clean.length) append(" …" )
223+ }
149224 }
150225
151226 private inner class HybridSession (
@@ -154,24 +229,86 @@ class HybridSearchEngine(
154229 private val bookIds : Collection <Long >? ,
155230 private val baseOnly : Boolean ,
156231 ) : SearchSession {
157- private var fused: List < LineHit > ? = null
232+ private var fused: FusedResult ? = null
158233 private var offset = 0
159234
160235 override suspend fun nextPage (limit : Int ): SearchPage ? {
161236 val all = fused ? : fuse(query, near, bookIds, baseOnly).also { fused = it }
162- if (offset >= all.size) return null
163- val end = minOf(offset + limit, all.size)
164- val slice = all.subList(offset, end)
237+ if (offset >= all.hits.size) return null
238+ val end = minOf(offset + limit, all.hits.size)
239+ // Re-snippet only the dense-only hits on THIS page (bounded work) so semantic
240+ // results show the passage that actually matched, not the line's opening words.
241+ val slice = all.hits.subList(offset, end).map { hit ->
242+ if (hit.lineId in all.lexicalIds) {
243+ hit
244+ } else {
245+ semanticSnippet(query, hit.rawText, near)?.let { hit.copy(snippet = it) } ? : hit
246+ }
247+ }
165248 offset = end
166- return SearchPage (hits = slice, totalHits = all.size.toLong(), isLastPage = offset >= all.size)
249+ return SearchPage (hits = slice, totalHits = all.hits. size.toLong(), isLastPage = offset >= all.hits .size)
167250 }
168251
169252 override fun close () {}
170253 }
171254
255+ /* *
256+ * Splits [text] into passage candidates: first on strong clause delimiters (incl. the
257+ * Hebrew sof-pasuk ׃), then sliding word windows over any segment too long to localize.
258+ * Substrings are verbatim slices of [text] (original spacing preserved) so the caller
259+ * can match them back diacritic-insensitively. Tiny fragments are dropped to keep a
260+ * highlight that is a *passage*, not a stray word.
261+ */
262+ private fun splitClauses (text : String ): List <String > {
263+ val delimiters = charArrayOf(' .' , ' !' , ' ?' , ' :' , ' ;' , ' ׃' , ' \n ' )
264+ val segments = ArrayList <IntRange >()
265+ var start = 0
266+ for (i in text.indices) {
267+ if (text[i] in delimiters) {
268+ if (i > start) segments + = start.. i
269+ start = i + 1
270+ }
271+ }
272+ if (start < text.length) segments + = start until text.length
273+
274+ val out = ArrayList <String >()
275+ for (seg in segments) {
276+ val sub = text.substring(seg.first, seg.last + 1 )
277+ val wordRanges = WORD_RE .findAll(sub).map { it.range }.toList()
278+ if (wordRanges.size < MIN_PASSAGE_WORDS ) continue
279+ if (wordRanges.size <= MAX_PASSAGE_WORDS ) {
280+ out + = sub.substring(wordRanges.first().first, wordRanges.last().last + 1 )
281+ } else {
282+ // Segment too long to localize: slide word windows over the original slice.
283+ var w = 0
284+ while (w < wordRanges.size) {
285+ val from = wordRanges[w].first
286+ val toIdx = minOf(w + MAX_PASSAGE_WORDS - 1 , wordRanges.size - 1 )
287+ out + = sub.substring(from, wordRanges[toIdx].last + 1 )
288+ if (toIdx == wordRanges.size - 1 ) break
289+ w + = WINDOW_STRIDE_WORDS
290+ }
291+ }
292+ }
293+ return if (out .isEmpty()) listOf (text.trim()) else out
294+ }
295+
296+ private fun cosine (a : FloatArray , b : FloatArray ): Float {
297+ var s = 0f
298+ val n = minOf(a.size, b.size)
299+ for (i in 0 until n) s + = a[i] * b[i]
300+ return s
301+ }
302+
172303 companion object {
173304 private val logger = Logger .withTag(" HybridSearch" )
174305
306+ private val WORD_RE = Regex (" \\ S+" )
307+ private const val MIN_PASSAGE_WORDS = 3
308+ private const val MAX_PASSAGE_WORDS = 14
309+ private const val WINDOW_STRIDE_WORDS = 8
310+ private const val SNIPPET_CONTEXT = 90 // chars of context kept on each side of the passage
311+
175312 fun create (
176313 lexical : LuceneSearchEngine ,
177314 indexDir : Path ,
0 commit comments