knowpro: pipeline for NLP -> scoped queries (#1353)

* Commands: Answer generation etc. issues NLP -> scoped queries
* Pipeline for scoped query testing
* Refactor 
* Bugs

Author: umeshma

ts/examples/chat/src/memory/knowproMemory.ts

@@ -122,6 +122,7 @@ export async function createKnowproCommands(
     commands.kpAnswerTerms = answerTerms;
     commands.kpEntities = entities;
     commands.kpTopics = topics;
+    commands.kpTags = tags;
     commands.kpMessages = showMessages;
     commands.kpAbstractMessage = abstract;
     commands.kpAlias = addAlias;
@@ -571,10 +572,7 @@ export async function createKnowproCommands(
     }
 
     function topicsDef(): CommandMetadata {
-        return searchTermsDef(
-            "Search topics only in current conversation",
-            "topic",
-        );
+        return searchTermsDef("Search topics in current conversation", "topic");
     }
     commands.kpTopics.metadata = topicsDef();
     async function topics(args: string[]): Promise<void> {
@@ -602,6 +600,33 @@ export async function createKnowproCommands(
         }
     }
 
+    function tagsDef(): CommandMetadata {
+        return searchTermsDef("Search tags in current conversation", "tag");
+    }
+    commands.kpTags.metadata = tagsDef();
+    async function tags(args: string[]): Promise<void> {
+        const conversation = ensureConversationLoaded();
+        if (!conversation) {
+            return;
+        }
+        if (args.length > 0) {
+            args.push("--ktype");
+            args.push("tag");
+            await searchTerms(args);
+        } else {
+            if (conversation.semanticRefs !== undefined) {
+                const tagRefs = kp.filterCollection(
+                    conversation.semanticRefs,
+                    (sr) => sr.knowledgeType === "tag",
+                );
+                let tags = tagRefs.map((t) => (t.knowledge as kp.Topic).text);
+                let tagStrings = kp.mergeTopics(tags);
+                tagStrings.sort();
+                context.printer.writeList(tagStrings, { type: "ol" });
+            }
+        }
+    }
+
     function abstractDef(): CommandMetadata {
         return {
             description: "Return an abstract of the message",

ts/examples/chat/src/memory/knowproPrinter.ts

@@ -189,6 +189,9 @@ export class KnowProPrinter extends MemoryConsoleWriter {
             case "topic":
                 this.writeTopic(semanticRef.knowledge as kp.Topic);
                 break;
+            case "tag":
+                this.writeTag(semanticRef.knowledge as kp.Tag);
+                break;
         }
         return this;
     }

ts/packages/knowPro/src/knowledge.ts

@@ -7,7 +7,7 @@ import { error, Result } from "typechat";
 import { ChatModel } from "aiclient";
 import { createKnowledgeModel } from "./conversationIndex.js";
 import { BatchTask, runInBatches } from "./taskQueue.js";
-import { SearchSelectExpr } from "./interfaces.js";
+import { SearchSelectExpr, Tag } from "./interfaces.js";
 import {
     createOrMaxTermGroup,
     createOrTermGroup,
@@ -66,6 +66,14 @@ export function mergeTopics(topics: string[]): string[] {
     return [...mergedTopics.values()];
 }
 
+export function mergeTags(tags: Tag[]): string[] {
+    let mergedText = new Set<string>();
+    for (let tag of tags) {
+        mergedText.add(tag.text);
+    }
+    return [...mergedText.values()];
+}
+
 export async function extractKnowledgeForTextBatchQ(
     knowledgeExtractor: kpLib.KnowledgeExtractor,
     textBatch: string[],

ts/packages/knowPro/src/searchLang.ts

@@ -22,6 +22,7 @@ import {
 import {
     searchQueryFromLanguage,
     SearchQueryTranslator,
+    searchQueryWithScopeFromLanguage,
 } from "./searchQueryTranslator.js";
 import * as querySchema from "./searchQuerySchema.js";
 import * as querySchema2 from "./searchQuerySchema_v2.js";
@@ -182,16 +183,6 @@ export async function searchConversationWithLanguage(
         //
         return undefined;
     }
-
-    function createTextQueryOptions(
-        options: LanguageSearchOptions,
-    ): SearchOptions {
-        const ragOptions: SearchOptions = {
-            ...options,
-            ...options.fallbackRagOptions,
-        };
-        return ragOptions;
-    }
 }
 
 export type LanguageQueryExpr = {
@@ -1013,6 +1004,170 @@ export function compileSearchQuery2(
     return searchQueryExprs;
 }
 
+export async function searchQueryExprFromLanguage2(
+    conversation: IConversation,
+    translator: SearchQueryTranslator,
+    queryText: string,
+    options?: LanguageSearchOptions,
+    languageSearchFilter?: LanguageSearchFilter,
+    debugContext?: LanguageSearchDebugContext,
+): Promise<Result<LanguageQueryExpr>> {
+    const queryResult = await searchQueryWithScopeFromLanguage(
+        conversation,
+        translator,
+        queryText,
+        options?.modelInstructions,
+    );
+    if (queryResult.success) {
+        const query = queryResult.data;
+        if (debugContext) {
+            debugContext.searchQuery = query;
+        }
+        options ??= createLanguageSearchOptions();
+        const queryExpressions = compileSearchQuery2(
+            conversation,
+            query,
+            options.compileOptions,
+            languageSearchFilter,
+        );
+        return success({
+            queryText,
+            query,
+            queryExpressions,
+        });
+    }
+    return queryResult;
+}
+
+/**
+ * Search a conversation using natural language. Returns {@link ConversationSearchResult} containing
+ * relevant knowledge and messages.
+ *
+ * @param conversation - The conversation object to search within.
+ * @param searchText - The natural language search phrase.
+ * @param queryTranslator - Translates natural language to a {@link querySchema.SearchQuery} structured query.
+ * @param options - Optional search options.
+ * @param langSearchFilter - Optional filter options for the search.
+ * @param debugContext - Optional context for debugging the search process.
+ * @returns {ConversationSearchResult} Conversation search results.
+ */
+export async function searchConversationWithLanguage2(
+    conversation: IConversation,
+    searchText: string,
+    queryTranslator: SearchQueryTranslator,
+    options?: LanguageSearchOptions,
+    langSearchFilter?: LanguageSearchFilter,
+    debugContext?: LanguageSearchDebugContext,
+): Promise<Result<ConversationSearchResult[]>> {
+    options ??= createLanguageSearchOptions();
+    const langQueryResult = await searchQueryExprFromLanguage2(
+        conversation,
+        queryTranslator,
+        searchText,
+        options,
+        langSearchFilter,
+        debugContext,
+    );
+    if (!langQueryResult.success) {
+        return langQueryResult;
+    }
+    const searchQueryExprs = langQueryResult.data.queryExpressions;
+    if (debugContext) {
+        debugContext.searchQueryExpr = searchQueryExprs;
+        debugContext.usedSimilarityFallback = new Array<boolean>(
+            searchQueryExprs.length,
+        );
+        debugContext.usedSimilarityFallback.fill(false);
+    }
+    let fallbackQueryExpr = compileFallbackQuery(
+        langQueryResult.data.query,
+        options.compileOptions,
+        langSearchFilter,
+    );
+
+    const searchResults: ConversationSearchResult[] = [];
+    for (let i = 0; i < searchQueryExprs.length; ++i) {
+        const searchQuery = searchQueryExprs[i];
+        const fallbackQuery = fallbackQueryExpr
+            ? fallbackQueryExpr[i]
+            : undefined;
+        let queryResult = await runSearchQuery(
+            conversation,
+            searchQuery,
+            options,
+        );
+        if (!hasConversationResults(queryResult) && fallbackQuery) {
+            // Rerun the query but with verb matching turned off for scopes
+            queryResult = await runSearchQuery(
+                conversation,
+                fallbackQuery,
+                options,
+            );
+        }
+        //
+        // If no matches and fallback enabled... run the raw query
+        //
+        if (
+            !hasConversationResults(queryResult) &&
+            searchQuery.rawQuery &&
+            options.fallbackRagOptions
+        ) {
+            const textSearchOptions = createTextQueryOptions(options);
+            const ragMatches = await runSearchQueryTextSimilarity(
+                conversation,
+                fallbackQuery ?? searchQuery,
+                textSearchOptions,
+            );
+            if (ragMatches) {
+                searchResults.push(...ragMatches);
+                if (debugContext?.usedSimilarityFallback) {
+                    debugContext.usedSimilarityFallback![i] = true;
+                }
+            }
+        } else {
+            searchResults.push(...queryResult);
+        }
+    }
+    return success(searchResults);
+
+    //
+    // Scoping queries can be precise. However, there may be random variations in how LLMs
+    // translate some user utterances into queries.. .particularly verbs. They verbs
+    // may not match action verbs actually in the index.. related terms may not meet the similarity
+    // cutoff.
+    // If configured (compileOptions.exactScope == false), we can do a fallback query that does
+    // not enforce verb matching. This improves recall while still providing a reasonable level of scoping because it
+    //
+    function compileFallbackQuery(
+        query: querySchema.SearchQuery,
+        compileOptions: LanguageQueryCompileOptions,
+        langSearchFilter?: LanguageSearchFilter,
+    ): SearchQueryExpr[] | undefined {
+        const verbScope = compileOptions.verbScope;
+        //
+        // If no exact scope... and verbScope is not provided or true,
+        // then we can build a fallback query that is more forgiving
+        if (
+            !compileOptions.exactScope &&
+            (verbScope == undefined || verbScope)
+        ) {
+            return compileSearchQuery2(
+                conversation,
+                query,
+                {
+                    ...compileOptions,
+                    verbScope: false,
+                },
+                langSearchFilter,
+            );
+        }
+        //
+        // No fallback query currently possible
+        //
+        return undefined;
+    }
+}
+
 const Wildcard = "*";
 
 function isEntityTermArray(
@@ -1042,3 +1197,11 @@ function optimizeOrMax(termGroup: SearchTermGroup) {
     }
     return termGroup;
 }
+
+function createTextQueryOptions(options: LanguageSearchOptions): SearchOptions {
+    const ragOptions: SearchOptions = {
+        ...options,
+        ...options.fallbackRagOptions,
+    };
+    return ragOptions;
+}

ts/packages/knowPro/src/searchQueryTranslator.ts

@@ -6,6 +6,7 @@ import {
     PromptSection,
     Result,
     TypeChatLanguageModel,
+    error,
 } from "typechat";
 import * as querySchema from "./searchQuerySchema.js";
 import * as querySchema2 from "./searchQuerySchema_v2.js";
@@ -85,3 +86,21 @@ export async function searchQueryFromLanguage(
     const result = await queryTranslator.translate(text, queryContext);
     return result;
 }
+
+export async function searchQueryWithScopeFromLanguage(
+    conversation: IConversation,
+    queryTranslator: SearchQueryTranslator,
+    text: string,
+    promptPreamble?: PromptSection[],
+): Promise<Result<querySchema2.SearchQuery>> {
+    if (!queryTranslator.translateWithScope) {
+        return error("Scoped queries not supported");
+    }
+    const timeRange = getTimeRangePromptSectionForConversation(conversation);
+    let queryContext: PromptSection[] =
+        promptPreamble && promptPreamble.length > 0
+            ? [...promptPreamble, ...timeRange]
+            : timeRange;
+    const result = await queryTranslator.translateWithScope(text, queryContext);
+    return result;
+}