knowpro: Better diagnostics, replayability (#1310)

For debugging, testing and co-dev with Python:
* Automatically dump kpSearch and kpAnswer results with timestamps in
log folder
* kpSearch: Run previously saved, or hand edited NLP SearchQuery expr
from file
* Better search expression comparison - for diffs
* Save validation run report
* Some comments
* Wrapper API to run multiple searches

Author: umeshma

ts/examples/chat/src/memory/knowproMemory.ts

@@ -23,7 +23,13 @@ import {
     parseFreeAndNamedArguments,
     keyValuesFromNamedArgs,
 } from "../common.js";
-import { collections, dateTime, ensureDir } from "typeagent";
+import {
+    collections,
+    dateTime,
+    ensureDir,
+    isFilePath,
+    readJsonFile,
+} from "typeagent";
 import chalk from "chalk";
 import { KnowProPrinter } from "./knowproPrinter.js";
 import { createKnowproDataFrameCommands } from "./knowproDataFrame.js";
@@ -277,16 +283,38 @@ export async function createKnowproCommands(
             return;
         }
         const namedArgs = parseNamedArguments(args, searchDef());
+        let savedQuery: kp.querySchema.SearchQuery | undefined;
+        if (isFilePath(namedArgs.query)) {
+            const queryPath = namedArgs.query;
+            const savedContext = await loadSavedDebugContext(queryPath);
+            if (!savedContext) {
+                return;
+            }
+            namedArgs.query = savedContext.searchText;
+            savedQuery = savedContext.searchQuery;
+            if (savedQuery) {
+                context.printer.writeSearchQuery(savedQuery);
+            }
+        }
+
         const searchResponse = await kpTest.execSearchRequest(
             context,
             namedArgs,
+            savedQuery,
         );
         const searchResults = searchResponse.searchResults;
         const debugContext = searchResponse.debugContext;
         if (!searchResults.success) {
             context.printer.writeError(searchResults.message);
             return;
         }
+        // Log any new queries
+        if (!savedQuery) {
+            context.log.writeFile("kpSearch", {
+                searchText: debugContext.searchText,
+                searchQuery: debugContext.searchQuery,
+            });
+        }
         if (namedArgs.debug) {
             context.printer.writeInColor(chalk.gray, () => {
                 context.printer.writeLine();
@@ -359,14 +387,15 @@ export async function createKnowproCommands(
         );
         getAnswerRequest.searchResponse = searchResponse;
         getAnswerRequest.knowledgeTopK = options.entitiesTopK;
-        await kpTest.execGetAnswerRequest(
+        const answerResponse = await kpTest.execGetAnswerRequest(
             context,
             getAnswerRequest,
             (i: number, q: string, answer) => {
                 writeAnswer(i, answer, debugContext);
                 return;
             },
         );
+        context.log.writeFile("kpAnswer", answerResponse);
         context.printer.writeLine();
     }
 
@@ -754,7 +783,7 @@ export async function createKnowproCommands(
     function writeAnswer(
         queryIndex: number,
         answerResult: Result<kp.AnswerResponse>,
-        debugContext: AnswerDebugContext,
+        debugContext: kpTest.AnswerDebugContext,
     ) {
         context.printer.writeLine();
         if (answerResult.success) {
@@ -943,8 +972,21 @@ export async function createKnowproCommands(
         }
         return [undefined, ""];
     }
-}
 
-export interface AnswerDebugContext extends kp.LanguageSearchDebugContext {
-    searchText: string;
+    async function loadSavedDebugContext(
+        filePath: string,
+    ): Promise<kpTest.AnswerDebugContext | undefined> {
+        // Load a saved or logged query
+        const savedContext =
+            await loadObject<kpTest.AnswerDebugContext>(filePath);
+        return savedContext;
+    }
+
+    async function loadObject<T>(filePath: string) {
+        const obj = await readJsonFile<T>(filePath);
+        if (obj === undefined) {
+            context.printer.writeError(`${filePath} not found`);
+        }
+        return obj;
+    }
 }

ts/examples/chat/src/memory/knowproPrinter.ts

@@ -569,6 +569,11 @@ export class KnowProPrinter extends MemoryConsoleWriter {
         return this;
     }
 
+    public writeSearchQuery(searchQuery: kp.querySchema.SearchQuery) {
+        this.writeHeading("Search Query");
+        this.writeJson(searchQuery);
+    }
+
     public writeDebugContext(context: kp.LanguageSearchDebugContext) {
         if (context.searchQuery) {
             this.writeHeading("Search Query");

ts/examples/chat/src/memory/knowproTest.ts

@@ -18,12 +18,14 @@ import * as kpTest from "knowpro-test";
 import * as cm from "conversation-memory";
 import {
     changeFileExt,
+    ensureDir,
     getAbsolutePath,
     getFileName,
     htmlToMd,
     readAllText,
     simplifyHtml,
     simplifyText,
+    writeJsonFile,
 } from "typeagent";
 import chalk from "chalk";
 import { openai } from "aiclient";
@@ -203,8 +205,6 @@ export async function createKnowproTestCommands(
                 srcPath: argSourceFile(),
             },
             options: {
-                startAt: argNum("Start at this query", 0),
-                count: argNum("Number to run"),
                 verbose: argBool("Verbose error output", false),
             },
         };
@@ -219,27 +219,23 @@ export async function createKnowproTestCommands(
 
             const namedArgs = parseNamedArguments(args, verifySearchBatchDef());
             const srcPath = namedArgs.srcPath;
-            let errorCount = 0;
             const results = await kpTest.verifyLangSearchResultsBatch(
                 context,
                 srcPath,
                 (result, index, total) => {
                     context.printer.writeProgress(index + 1, total);
                     if (result.success) {
-                        if (!writeSearchScore(result.data, namedArgs.verbose)) {
-                            errorCount++;
-                        }
+                        writeSearchScore(result.data, namedArgs.verbose);
                     } else {
                         context.printer.writeError(result.message);
                     }
                 },
             );
             if (!results.success) {
                 context.printer.writeError(results.message);
+                return;
             }
-            if (errorCount > 0) {
-                context.printer.writeLine(`${errorCount} errors`);
-            }
+            await writeSearchValidationReport(results.data, srcPath);
         } finally {
             endTestBatch();
         }
@@ -442,11 +438,12 @@ export async function createKnowproTestCommands(
     ): boolean {
         const error = result.error;
         if (error !== undefined && error.length > 0) {
-            context.printer.writeInColor(
-                chalk.redBright,
-                `[${error}]: ${result.expected.cmd!}`,
+            context.printer.writeLineInColor(
+                chalk.gray,
+                result.actual.searchText,
             );
-            context.printer.writeInColor(chalk.red, `Error: ${error}`);
+            context.printer.writeInColor(chalk.red, `${result.expected.cmd!}`);
+            context.printer.writeInColor(chalk.red, `Error:\n ${error}`);
             if (verbose) {
                 context.printer.writeLine("===========");
                 context.printer.writeJsonInColor(
@@ -477,6 +474,25 @@ export async function createKnowproTestCommands(
         }
     }
 
+    async function writeSearchValidationReport(
+        results: kpTest.Comparison<kpTest.LangSearchResults>[],
+        srcPath?: string,
+    ) {
+        const errorResults = results.filter(
+            (c) => c.error !== undefined && c.error.length > 0,
+        );
+        if (errorResults.length === 0) {
+            context.printer.writeLineInColor(chalk.green, "No errors");
+            return;
+        }
+
+        context.printer.writeLine(`${errorResults.length} errors`);
+        context.printer.writeList(errorResults.map((e) => e.expected.cmd));
+        if (srcPath) {
+            await saveReport(srcPath, errorResults);
+        }
+    }
+
     function writeAnswerScore(
         result: kpTest.SimilarityComparison<kpTest.QuestionAnswer>,
         threshold: number,
@@ -498,6 +514,13 @@ export async function createKnowproTestCommands(
         }
     }
 
+    async function saveReport(srcPath: string, report: any) {
+        const outputDir = path.join(context.basePath, "logs/testReports");
+        await ensureDir(outputDir);
+        const outputPath = path.join(outputDir, getFileName(srcPath) + ".json");
+        await writeJsonFile(outputPath, report);
+    }
+
     function beginTestBatch() {
         context.retryNoAnswer = true;
     }

ts/packages/knowPro/src/search.ts

@@ -228,6 +228,27 @@ export async function runSearchQuery(
     return results;
 }
 
+/**
+ * Run multiple queries
+ * @param conversation
+ * @param queries queries to run
+ * @param options
+ * @returns
+ */
+export async function runSearchQueries(
+    conversation: IConversation,
+    queries: SearchQueryExpr[],
+    options?: SearchOptions,
+): Promise<ConversationSearchResult[][]> {
+    // FUTURE: do these in parallel
+    const results: ConversationSearchResult[][] = [];
+    for (let i = 0; i < queries.length; ++i) {
+        const result = await runSearchQuery(conversation, queries[i], options);
+        results.push(result);
+    }
+    return results;
+}
+
 /**
  * Run the search query. For each selectExpr:
  * - only match messages using similarity to the rawQuery on each expression

ts/packages/knowPro/src/searchLang.ts

@@ -145,12 +145,23 @@ export async function searchConversationWithLanguage(
     }
     return success(searchResults);
 
+    //
+    // Scoping queries can be precise. However, there may be random variations in how LLMs
+    // translate some user utterances into queries.. .particularly verbs. They verbs
+    // may not match action verbs actually in the index.. related terms may not meet the similarity
+    // cutoff.
+    // If configured (compileOptions.exactScope == false), we can do a fallback query that does
+    // not enforce verb matching. This improves recall while still providing a reasonable level of scoping because it
+    //
     function compileFallbackQuery(
         query: querySchema.SearchQuery,
         compileOptions: LanguageQueryCompileOptions,
         langSearchFilter?: LanguageSearchFilter,
     ): SearchQueryExpr[] | undefined {
         const verbScope = compileOptions.verbScope;
+        //
+        // If no exact scope... and verbScope is not provided or true,
+        // then we can build a fallback query that is more forgiving
         if (
             !compileOptions.exactScope &&
             (verbScope == undefined || verbScope)
@@ -165,6 +176,9 @@ export async function searchConversationWithLanguage(
                 langSearchFilter,
             );
         }
+        //
+        // No fallback query currently possible
+        //
         return undefined;
     }
 
@@ -249,6 +263,9 @@ export type LanguageQueryCompileOptions = {
      * Is fuzzy matching enabled when applying scope?
      */
     exactScope?: boolean | undefined;
+    /**
+     * Should use verbs in scoping expressions (default true)
+     */
     verbScope?: boolean | undefined;
     // Use to ignore noise terms etc.
     termFilter?: (text: string) => boolean;

ts/packages/knowProTest/src/common.ts

@@ -14,6 +14,26 @@ export function ensureDirSync(folderPath: string): string {
     return folderPath;
 }
 
+export function ensureUniqueFilePath(filePath: string): string {
+    if (!fs.existsSync(filePath)) {
+        return filePath;
+    }
+
+    for (let i = 1; i < 1000; ++i) {
+        const tempPath = addFileNameSuffixToPath(filePath, `_${i}`);
+        if (!fs.existsSync(tempPath)) {
+            return tempPath;
+        }
+    }
+
+    throw new Error(`Could not ensure unique ${filePath}`);
+}
+
+export function writeObjectToUniqueFile(filePath: string, obj: any): void {
+    filePath = ensureUniqueFilePath(filePath);
+    fs.writeFileSync(filePath, stringifyReadable(obj));
+}
+
 export function getCommandArgs(line: string | undefined): string[] {
     if (line !== undefined && line.length > 0) {
         const args = parseCommandLine(line);
@@ -102,6 +122,17 @@ export function isJsonEqual(x: any | undefined, y: any | undefined): boolean {
     return false;
 }
 
+export function compareObject(
+    x: any,
+    y: any,
+    label: string,
+): string | undefined {
+    if (!isJsonEqual(x, y)) {
+        return `${label}: ${stringifyReadable(x)}\n !== \n${stringifyReadable(y)}`;
+    }
+    return undefined;
+}
+
 export function compareArray(
     name: string,
     x: any[] | undefined,
@@ -129,3 +160,7 @@ export function compareArray(
 export function queryError(query: string, result: Error): Error {
     return error(`${query}\n${result.message}`);
 }
+
+export function stringifyReadable(value: any): string {
+    return JSON.stringify(value, undefined, 2);
+}

ts/packages/knowProTest/src/index.ts

@@ -3,6 +3,7 @@
 
 export * from "./types.js";
 export * from "./models.js";
+export * from "./logging.js";
 export * from "./knowproContext.js";
 export * from "./searchTest.js";
 export * from "./answerTest.js";