knowproTest improvements (#1314)

Improved test result collection, logging

Author: umeshma

ts/examples/chat/src/memory/knowproMemory.ts

@@ -310,7 +310,7 @@ export async function createKnowproCommands(
         }
         // Log any new queries
         if (!savedQuery) {
-            context.log.writeFile("kpSearch", {
+            context.log.writeCommandResult("kpSearch", {
                 searchText: debugContext.searchText,
                 searchQuery: debugContext.searchQuery,
             });
@@ -395,7 +395,7 @@ export async function createKnowproCommands(
                 return;
             },
         );
-        context.log.writeFile("kpAnswer", answerResponse);
+        context.log.writeCommandResult("kpAnswer", answerResponse);
         context.printer.writeLine();
     }
 

ts/examples/chat/src/memory/knowproTest.ts

@@ -18,14 +18,12 @@ import * as kpTest from "knowpro-test";
 import * as cm from "conversation-memory";
 import {
     changeFileExt,
-    ensureDir,
     getAbsolutePath,
     getFileName,
     htmlToMd,
     readAllText,
     simplifyHtml,
     simplifyText,
-    writeJsonFile,
 } from "typeagent";
 import chalk from "chalk";
 import { openai } from "aiclient";
@@ -192,6 +190,7 @@ export async function createKnowproTestCommands(
             );
             if (!results.success) {
                 context.printer.writeError(results.message);
+                return;
             }
         } finally {
             endTestBatch();
@@ -219,6 +218,8 @@ export async function createKnowproTestCommands(
 
             const namedArgs = parseNamedArguments(args, verifySearchBatchDef());
             const srcPath = namedArgs.srcPath;
+
+            const startTimestamp = new Date();
             const results = await kpTest.verifyLangSearchResultsBatch(
                 context,
                 srcPath,
@@ -235,7 +236,11 @@ export async function createKnowproTestCommands(
                 context.printer.writeError(results.message);
                 return;
             }
-            await writeSearchValidationReport(results.data, srcPath);
+            await writeSearchValidationReport(
+                results.data,
+                srcPath,
+                startTimestamp,
+            );
         } finally {
             endTestBatch();
         }
@@ -476,21 +481,18 @@ export async function createKnowproTestCommands(
 
     async function writeSearchValidationReport(
         results: kpTest.Comparison<kpTest.LangSearchResults>[],
-        srcPath?: string,
+        srcPath: string,
+        startDate: Date,
     ) {
-        const errorResults = results.filter(
-            (c) => c.error !== undefined && c.error.length > 0,
-        );
-        if (errorResults.length === 0) {
-            context.printer.writeLineInColor(chalk.green, "No errors");
-            return;
-        }
-
-        context.printer.writeLine(`${errorResults.length} errors`);
-        context.printer.writeList(errorResults.map((e) => e.expected.cmd));
-        if (srcPath) {
-            await saveReport(srcPath, errorResults);
+        const report = kpTest.createSearchTestReport(results, srcPath, {
+            startDate,
+            stopDate: new Date(),
+        });
+        context.printer.writeLine(`${report.errors.length} errors`);
+        if (report.errors.length > 0) {
+            context.printer.writeList(report.errors.map((e) => e.expected.cmd));
         }
+        await context.log.writeTestReport(report, getFileName(srcPath));
     }
 
     function writeAnswerScore(
@@ -514,17 +516,11 @@ export async function createKnowproTestCommands(
         }
     }
 
-    async function saveReport(srcPath: string, report: any) {
-        const outputDir = path.join(context.basePath, "logs/testReports");
-        await ensureDir(outputDir);
-        const outputPath = path.join(outputDir, getFileName(srcPath) + ".json");
-        await writeJsonFile(outputPath, report);
-    }
-
-    function beginTestBatch() {
+    function beginTestBatch(): void {
         context.retryNoAnswer = true;
     }
-    function endTestBatch() {
+
+    function endTestBatch(): void {
         context.retryNoAnswer = false;
     }
     return;

ts/packages/knowProTest/src/common.ts

@@ -4,7 +4,7 @@
 import fs from "fs";
 import { ArgDef, NamedArgs, parseCommandLine } from "interactive-app";
 import path from "path";
-import { getFileName } from "typeagent";
+import { dateTime, getFileName } from "typeagent";
 import { error, Result, Error } from "typechat";
 
 export function ensureDirSync(folderPath: string): string {
@@ -34,6 +34,10 @@ export function writeObjectToUniqueFile(filePath: string, obj: any): void {
     fs.writeFileSync(filePath, stringifyReadable(obj));
 }
 
+export function writeObjectToFile(filePath: string, obj: any): void {
+    fs.writeFileSync(filePath, stringifyReadable(obj));
+}
+
 export function getCommandArgs(line: string | undefined): string[] {
     if (line !== undefined && line.length > 0) {
         const args = parseCommandLine(line);
@@ -47,6 +51,15 @@ export function getCommandArgs(line: string | undefined): string[] {
     return [];
 }
 
+export function dateRangeToTimeRange(
+    range: dateTime.DateRange,
+): dateTime.TimestampRange {
+    return {
+        startTimestamp: range.startDate.toISOString(),
+        endTimestamp: range.stopDate ? range.stopDate.toISOString() : undefined,
+    };
+}
+
 export async function execCommandLine<T>(
     line: string,
     cb: (args: string[], cmdLine: string) => Promise<Result<T>>,

ts/packages/knowProTest/src/logging.ts

@@ -1,19 +1,23 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
-import { ensureDirSync, writeObjectToUniqueFile } from "./common.js";
-import { dateTime } from "typeagent";
+import {
+    ensureDirSync,
+    writeObjectToFile,
+    writeObjectToUniqueFile,
+} from "./common.js";
+import { changeFileExt, dateTime, ensureDir } from "typeagent";
 import path from "path";
 
 export class KnowproLog {
     constructor(public baseLogDir: string) {
         ensureDirSync(baseLogDir);
     }
 
-    public writeFile(commandName: string, obj: any) {
+    public writeCommandResult(commandName: string, obj: any) {
         try {
             const timestamp = new Date();
-            const dirPath = this.ensureLogDir(timestamp, commandName);
+            const dirPath = this.ensureCommandLogDir(timestamp, commandName);
             const fileName = `${dateTime.timestampString(timestamp)}.json`;
             let filePath = path.join(dirPath, fileName);
             writeObjectToUniqueFile(filePath, obj);
@@ -22,7 +26,27 @@ export class KnowproLog {
         }
     }
 
-    public ensureLogDir(timestamp: Date, commandName?: string): string {
+    public async writeTestReport(
+        report: TestRunReport,
+        fileName: string,
+        timestamp?: Date,
+    ) {
+        const logDir = path.join(this.baseLogDir, "testReports");
+        await ensureDir(logDir);
+
+        timestamp ??= new Date();
+        const outputPath = path.join(
+            logDir,
+            changeFileExt(
+                fileName,
+                ".json",
+                dateTime.timestampString(timestamp),
+            ),
+        );
+        writeObjectToFile(outputPath, report);
+    }
+
+    private ensureCommandLogDir(timestamp: Date, commandName?: string): string {
         let dirName: string;
         if (commandName) {
             dirName = `${commandName}_${dateTime.timestampStringShort(timestamp)}`;
@@ -34,3 +58,12 @@ export class KnowproLog {
         return dirPath;
     }
 }
+
+export interface TestRunReport<T = any> {
+    name: string;
+    timeRange: dateTime.TimestampRange;
+    srcData: string;
+    countRun: number;
+    errors: T[];
+    rawResults?: T[] | undefined;
+}

ts/packages/knowProTest/src/searchTest.ts

@@ -2,24 +2,27 @@
 // Licensed under the MIT License.
 
 import { KnowproContext } from "./knowproContext.js";
-import { readJsonFile, writeJsonFile } from "typeagent";
+import { dateTime, readJsonFile, writeJsonFile } from "typeagent";
 import { error, Result, success } from "typechat";
 import { BatchCallback, Comparison } from "./types.js";
 import {
     compareArray,
     compareObject,
+    dateRangeToTimeRange,
     getCommandArgs,
     queryError,
 } from "./common.js";
 import { getLangSearchResult } from "./knowproCommands.js";
 import { getBatchFileLines } from "interactive-app";
 import { execSearchRequest } from "./knowproCommands.js";
 import * as kp from "knowpro";
+import { TestRunReport } from "./logging.js";
 
 export type LangSearchResults = {
     searchText: string;
     cmd?: string | undefined;
     searchQueryExpr: kp.querySchema.SearchQuery;
+    compiledQueryExpr?: kp.SearchQueryExpr[] | undefined;
     results: LangSearchResult[];
 };
 
@@ -65,7 +68,6 @@ export async function runSearchBatch(
     }
     return success(results);
 }
-
 async function getSearchResults(
     context: KnowproContext,
     args: string[],
@@ -90,6 +92,7 @@ function collectLangSearchResults(
     return {
         searchText,
         searchQueryExpr: debugContext.searchQuery!,
+        compiledQueryExpr: debugContext.searchQueryExpr,
         results: searchResults.map((cr) => {
             const lr: LangSearchResult = {
                 messageMatches: cr.messageMatches.map((m) => m.messageOrdinal),
@@ -198,6 +201,23 @@ export async function* runLangSearchBatch(
     }
 }
 
+export function createSearchTestReport(
+    results: Comparison<LangSearchResults>[],
+    srcDataPath: string,
+    dateRange: dateTime.DateRange,
+): TestRunReport<Comparison<LangSearchResults>> {
+    const errors = results.filter(
+        (c) => c.error !== undefined && c.error.length > 0,
+    );
+    return {
+        name: "search",
+        timeRange: dateRangeToTimeRange(dateRange),
+        srcData: srcDataPath,
+        errors,
+        countRun: results.length,
+    };
+}
+
 function getKnowledgeResults(
     cr: kp.ConversationSearchResult,
     lr: LangSearchResult,