DocMemory, knowpro (#1366)

* SearchTermGroup validation
  * Unit Tests 
* Allow for experimentation with custom query schemas 
* DocMemory
  * Scope query improvements 
  * Experimental schema with custom instructions for scoped queries

Author: umeshma

ts/.vscode/launch.json

@@ -329,9 +329,9 @@
         "--runTestsByPath",
         "${jest.testFile}"
       ],
-      //"cwd": "${workspaceFolder}/packages/knowpro",
+      "cwd": "${workspaceFolder}/packages/knowpro",
       //"cwd": "${workspaceFolder}/packages/memory/storage",
-      "cwd": "${workspaceFolder}/packages/memory/conversation",
+      //"cwd": "${workspaceFolder}/packages/memory/conversation",
       //"cwd": "${workspaceFolder}/packages/knowledgeProcessor",
       //"cwd": "${workspaceFolder}/packages/aiclient",
       "console": "integratedTerminal",

ts/examples/chat/src/memory/knowproTest.ts

@@ -449,26 +449,41 @@ export async function createKnowproTestCommands(
         return undefined;
     }
 
-    commands.kpTestSearch.metadata = searchDef();
+    function testSearchDef(): CommandMetadata {
+        const def = searchDef();
+        def.options ??= {};
+        def.options!.compare = argBool("Compare to V1 mode", false);
+        return def;
+    }
+    commands.kpTestSearch.metadata = testSearchDef();
     async function testSearchScope(args: string[]) {
-        const namedArgs = parseNamedArguments(args, searchDef());
-        const result = await context.queryTranslator.translate!(
-            namedArgs.query,
-        );
-        context.printer.writeTranslation(result);
+        const namedArgs = parseNamedArguments(args, testSearchDef());
+        const queryTranslator =
+            context.conversation! instanceof cm.Memory
+                ? context.conversation!.settings.queryTranslator
+                : context.queryTranslator;
+        const result = namedArgs.compare
+            ? await queryTranslator.translate(namedArgs.query)
+            : undefined;
+        if (result) {
+            context.printer.writeTranslation(result);
+        }
         context.printer.writeHeading("Scope");
-        const resultScope = await context.queryTranslator.translateWithScope!(
+        const resultScope = await queryTranslator.translateWithScope!(
             namedArgs.query,
         );
         context.printer.writeTranslation(resultScope);
-        if (result.success && resultScope.success) {
-            const error = kpTest.compareSearchQueryScope(
-                result.data,
-                resultScope.data,
-            );
-            if (error !== undefined) {
-                context.printer.writeError(error);
+        if ((result === undefined || result.success) && resultScope.success) {
+            if (result !== undefined) {
+                const error = kpTest.compareSearchQueryScope(
+                    result.data,
+                    resultScope.data,
+                );
+                if (error !== undefined) {
+                    context.printer.writeError(error);
+                }
             }
+
             const request = parseTypedArguments<kpTest.SearchRequest>(
                 args,
                 searchDef(),

ts/packages/knowPro/src/compileLib.ts

@@ -12,6 +12,8 @@ import {
     SearchTerm,
     PropertySearchTerm,
     SearchTermGroup,
+    Term,
+    SearchTermGroupTypes,
 } from "./interfaces.js";
 import * as q from "./query.js";
 
@@ -99,6 +101,89 @@ export function isSearchGroupTerm(
     return term.hasOwnProperty("booleanOp");
 }
 
+export function validateSearchTermGroup(
+    termGroup: SearchTermGroup,
+): string | undefined {
+    if (termGroup === undefined) {
+        return "SearchTermGroup";
+    }
+    if (!termGroup.booleanOp) {
+        return "booleanOp";
+    }
+    if (termGroup.terms === undefined || termGroup.terms.length === 0) {
+        return `terms\n${JSON.stringify(termGroup)}`;
+    }
+    let error: string | undefined;
+    let lastValid: SearchTermGroupTypes | undefined;
+    for (let i = 0; i < termGroup.terms.length; ++i) {
+        const term = termGroup.terms[i];
+        if (isPropertyTerm(term)) {
+            if (term.propertyName === undefined) {
+                error = `propertyName\n${JSON.stringify(term)}`;
+                break;
+            }
+            if (typeof term.propertyName !== "string") {
+                error = validateSearchTerm(term.propertyName);
+                if (error !== undefined) {
+                    error = "propertyName\n" + error;
+                    break;
+                }
+            }
+            if (term.propertyValue === undefined) {
+                error = `propertyValue\n${JSON.stringify(term)}`;
+                break;
+            }
+            error = validateSearchTerm(term.propertyValue);
+            if (error !== undefined) {
+                error = "propertyValue\n" + error;
+                break;
+            }
+        } else if (isSearchGroupTerm(term)) {
+            error = validateSearchTermGroup(term);
+        } else {
+            error = validateSearchTerm(term);
+        }
+        if (error !== undefined) {
+            break;
+        }
+        lastValid = term;
+    }
+    if (error !== undefined && lastValid !== undefined) {
+        error += `\nLast valid term:\n${JSON.stringify(lastValid)}`;
+    }
+    return error;
+}
+
+function validateSearchTerm(term: SearchTerm): string | undefined {
+    if (!validateTerm(term.term)) {
+        return `Invalid SearchTerm\n${JSON.stringify(term)}`;
+    }
+    if (term.relatedTerms !== undefined && term.relatedTerms.length > 0) {
+        for (let i = 0; i < term.relatedTerms.length; ++i) {
+            let relatedTerm = term.relatedTerms[i];
+            if (!validateTerm(relatedTerm)) {
+                let error = `Invalid related term for:\n${JSON.stringify(term.term)}`;
+                if (i > 0) {
+                    error += `\nLast valid related term:\n${JSON.stringify(term.relatedTerms[i - 1])}`;
+                }
+                return error;
+            }
+        }
+    }
+    return undefined;
+}
+
+function validateTerm(term: Term): boolean {
+    if (
+        term === undefined ||
+        term.text === undefined ||
+        term.text.length === 0
+    ) {
+        return false;
+    }
+    return true;
+}
+
 export interface CompiledSearchTerm extends SearchTerm {
     /**
      * The compiler will eliminate overlapping related terms to reduce hit duplication.

ts/packages/knowPro/src/search.ts

@@ -39,6 +39,7 @@ import {
     isPropertyTerm,
     isSearchGroupTerm,
     toRequiredSearchTerm,
+    validateSearchTermGroup,
 } from "./compileLib.js";
 import { NormalizedEmbedding } from "typeagent";
 import { getTimestampedScoredSemanticRefOrdinals } from "./knowledgeLib.js";
@@ -151,6 +152,11 @@ export async function searchConversationKnowledge(
     if (!q.isConversationSearchable(conversation)) {
         return undefined;
     }
+    let error = validateSearchTermGroup(searchTermGroup);
+    if (error !== undefined) {
+        throw new Error(error);
+    }
+
     options ??= createSearchOptions();
     const queryBuilder = new QueryCompiler(
         conversation,
@@ -816,7 +822,11 @@ class QueryCompiler {
      * Return false if the term should be rejected
      */
     private validateAndPrepareTerm(term: Term | undefined): boolean {
-        if (term) {
+        if (
+            term !== undefined &&
+            term.text !== undefined &&
+            term.text.length > 0
+        ) {
             term.text = term.text.toLowerCase();
         }
         return true;

ts/packages/knowPro/src/searchLang.ts

@@ -974,14 +974,29 @@ class SearchQueryCompiler {
     private compileScopeFilter(
         filter: querySchema2.ScopeFilter,
         when: WhenFilter | undefined,
+        useTags: boolean = true,
     ): WhenFilter | undefined {
         when ??= {};
         if (filter.timeRange) {
             when.dateRange = dateRangeFromDateTimeRange(filter.timeRange);
         }
         if (filter.searchTerms !== undefined && filter.searchTerms.length > 0) {
             when.tags ??= [];
-            when.tags.push(...filter.searchTerms);
+            if (useTags) {
+                when.tags.push(...filter.searchTerms);
+            } else {
+                const topicTerms = createOrMaxTermGroup();
+                for (const topic of filter.searchTerms) {
+                    topicTerms.terms.push(
+                        createPropertySearchTerm("topic", topic),
+                    );
+                }
+                if (when.scopeDefiningTerms) {
+                    when.scopeDefiningTerms.terms.push(topicTerms);
+                } else {
+                    when.scopeDefiningTerms = topicTerms;
+                }
+            }
         }
         return when;
     }

ts/packages/knowPro/src/searchQueryTranslator.ts

@@ -7,6 +7,7 @@ import {
     Result,
     TypeChatLanguageModel,
     error,
+    TypeChatJsonTranslator,
 } from "typechat";
 import * as querySchema from "./searchQuerySchema.js";
 import * as querySchema2 from "./searchQuerySchema_v2.js";
@@ -38,30 +39,15 @@ export interface SearchQueryTranslator {
 export function createSearchQueryTranslator(
     model: TypeChatLanguageModel,
 ): SearchQueryTranslator {
-    const typeName = "SearchQuery";
-    const searchActionSchema = loadSchema(
-        ["dateTimeSchema.ts", "searchQuerySchema.ts"],
-        import.meta.url,
-    );
-    const searchActionSchemaScope = loadSchema(
-        ["dateTimeSchema.ts", "searchQuerySchema_v2.ts"],
-        import.meta.url,
-    );
-
-    const translator = createJsonTranslator<querySchema.SearchQuery>(
-        model,
-        createTypeScriptJsonValidator<querySchema.SearchQuery>(
-            searchActionSchema,
-            typeName,
-        ),
-    );
-    const translator_V2 = createJsonTranslator<querySchema2.SearchQuery>(
+    const translator = createSearchQueryJsonTranslator<querySchema.SearchQuery>(
         model,
-        createTypeScriptJsonValidator<querySchema2.SearchQuery>(
-            searchActionSchemaScope,
-            typeName,
-        ),
+        "searchQuerySchema.ts",
     );
+    const translator_V2 =
+        createSearchQueryJsonTranslator<querySchema2.SearchQuery>(
+            model,
+            "searchQuerySchema_v2.ts",
+        );
     return {
         translate(request, promptPreamble) {
             return translator.translate(request, promptPreamble);
@@ -72,6 +58,29 @@ export function createSearchQueryTranslator(
     };
 }
 
+/**
+ * Create a query translator using
+ * @param {TypeChatLanguageModel} model
+ * @param schemaFilePath Relative path to schema file
+ * @returns {SearchQueryTranslator}
+ */
+export function createSearchQueryJsonTranslator<
+    T extends querySchema.SearchQuery | querySchema2.SearchQuery,
+>(
+    model: TypeChatLanguageModel,
+    schemaFilePath: string,
+): TypeChatJsonTranslator<T> {
+    const typeName = "SearchQuery";
+    const searchActionSchema = loadSchema(
+        ["dateTimeSchema.ts", schemaFilePath],
+        import.meta.url,
+    );
+    return createJsonTranslator<T>(
+        model,
+        createTypeScriptJsonValidator<T>(searchActionSchema, typeName),
+    );
+}
+
 export async function searchQueryFromLanguage(
     conversation: IConversation,
     queryTranslator: SearchQueryTranslator,