knowpro: distinct operations (#745)

Ongoing iteration on 'distinct' operator.
microsoft · Feb 24, 2025 · dd7a7b1 · dd7a7b1
1 parent c2e98ac
commit dd7a7b1
Show file tree

Hide file tree

Showing 4 changed files with 138 additions and 6 deletions.
diff --git a/ts/examples/chat/src/memory/knowproMemory.ts b/ts/examples/chat/src/memory/knowproMemory.ts
@@ -238,6 +238,7 @@ export async function createKnowproCommands(
                     "Use timestamp index while searching",
                     true,
                 ),
+                distinct: argBool("Show distinct results", false),
             },
         };
         if (kType === undefined) {
@@ -290,6 +291,7 @@ export async function createKnowproCommands(
                     conversation,
                     matches,
                     namedArgs.maxToDisplay,
+                    namedArgs.distinct,
                 );
             } else {
                 context.printer.writeLine("No matches");

diff --git a/ts/examples/chat/src/memory/knowproPrinter.ts b/ts/examples/chat/src/memory/knowproPrinter.ts
@@ -175,12 +175,28 @@ export class KnowProPrinter extends ChatPrinter {
         conversation: kp.IConversation,
         results: Map<kp.KnowledgeType, kp.SearchResult>,
         maxToDisplay: number,
+        distinct: boolean = false,
     ) {
-        // Do entities before actions...
-        this.writeResult(conversation, "entity", results, maxToDisplay);
-        this.writeResult(conversation, "action", results, maxToDisplay);
-        this.writeResult(conversation, "topic", results, maxToDisplay);
-        this.writeResult(conversation, "tag", results, maxToDisplay);
+        if (distinct) {
+            this.writeResultDistinct(
+                conversation,
+                "entity",
+                results,
+                maxToDisplay,
+            );
+            this.writeResultDistinct(
+                conversation,
+                "topic",
+                results,
+                maxToDisplay,
+            );
+        } else {
+            // Do entities before actions...
+            this.writeResult(conversation, "entity", results, maxToDisplay);
+            this.writeResult(conversation, "action", results, maxToDisplay);
+            this.writeResult(conversation, "topic", results, maxToDisplay);
+            this.writeResult(conversation, "tag", results, maxToDisplay);
+        }
         return this;
     }
 
@@ -198,6 +214,79 @@ export class KnowProPrinter extends ChatPrinter {
         return this;
     }
 
+    private writeResultDistinct(
+        conversation: kp.IConversation,
+        type: kp.KnowledgeType,
+        results: Map<kp.KnowledgeType, kp.SearchResult>,
+        maxToDisplay: number,
+    ) {
+        if (this.sortAsc) {
+            this.writeLine(`Sorted in ascending order (lowest first)`);
+        }
+        switch (type) {
+            default:
+                return;
+
+            case "topic":
+                const topics = results.get("topic");
+                if (topics) {
+                    let distinctTopics = kp.getDistinctTopicMatches(
+                        conversation.semanticRefs!,
+                        topics.semanticRefMatches,
+                        maxToDisplay,
+                    );
+                    for (let i = 0; i < distinctTopics.length; ++i) {
+                        let pos = this.sortAsc
+                            ? distinctTopics.length - (i + 1)
+                            : i;
+                        const entity = distinctTopics[pos];
+                        this.writeInColor(
+                            chalk.green,
+                            `#${pos + 1} / ${distinctTopics.length}: [${entity.score}]`,
+                        );
+                        this.writeLine(entity.item.text);
+                        this.writeLine();
+                    }
+                }
+                break;
+
+            case "entity":
+                const entities = results.get("entity");
+                if (entities) {
+                    let distinctEntities = kp.getDistinctEntityMatches(
+                        conversation.semanticRefs!,
+                        entities.semanticRefMatches,
+                        maxToDisplay,
+                    );
+                    for (let i = 0; i < distinctEntities.length; ++i) {
+                        let pos = this.sortAsc
+                            ? distinctEntities.length - (i + 1)
+                            : i;
+                        const entity = distinctEntities[pos];
+                        this.writeInColor(
+                            chalk.green,
+                            `#${pos + 1} / ${distinctEntities.length}: [${entity.score}]`,
+                        );
+                        this.writeCompositeEntity(entity.item);
+                        this.writeLine();
+                    }
+                }
+                break;
+        }
+
+        return this;
+    }
+
+    public writeCompositeEntity(
+        entity: knowLib.conversation.CompositeEntity | undefined,
+    ): void {
+        if (entity) {
+            this.writeLine(entity.name.toUpperCase());
+            this.writeList(entity.type, { type: "csv" });
+            this.writeList(entity.facets, { type: "ul" });
+        }
+    }
+
     public writeConversationInfo(conversation: kp.IConversation) {
         this.writeTitle(conversation.nameTag);
         const timeRange = kp.getTimeRangeForConversation(conversation);

diff --git a/ts/packages/knowPro/src/query.ts b/ts/packages/knowPro/src/query.ts
@@ -15,6 +15,7 @@ import {
     Term,
     TextLocation,
     TextRange,
+    Topic,
 } from "./dataFormat.js";
 import {
     CompositeEntity,
@@ -1175,7 +1176,9 @@ export function mergeEntityMatches(
         const existing = mergedEntities.get(compositeEntity.name);
         if (existing) {
             if (combineCompositeEntities(existing.item, compositeEntity)) {
-                existing.score += semanticRefMatch.score;
+                if (existing.score < semanticRefMatch.score) {
+                    existing.score = semanticRefMatch.score;
+                }
             }
         } else {
             mergedEntities.set(compositeEntity.name, {
@@ -1227,3 +1230,33 @@ function combineCompositeEntities(
     x.facets = unionArrays(x.facets, y.facets);
     return true;
 }
+
+export function mergeTopics(
+    semanticRefs: SemanticRef[],
+    semanticRefMatches: ScoredSemanticRef[],
+    topK?: number,
+): Scored<Topic>[] {
+    let mergedTopics = new Map<string, Scored<Topic>>();
+    for (let semanticRefMatch of semanticRefMatches) {
+        const semanticRef = semanticRefs[semanticRefMatch.semanticRefIndex];
+        if (semanticRef.knowledgeType !== "topic") {
+            continue;
+        }
+        const topic = semanticRef.knowledge as Topic;
+        const existing = mergedTopics.get(topic.text);
+        if (existing) {
+            if (existing.score < semanticRefMatch.score) {
+                existing.score = semanticRefMatch.score;
+            }
+        } else {
+            mergedTopics.set(topic.text, {
+                item: topic,
+                score: semanticRefMatch.score,
+            });
+        }
+    }
+    if (topK !== undefined && topK > 0) {
+        return getTopK(mergedTopics.values(), topK);
+    }
+    return [...mergedTopics.values()];
+}
diff --git a/ts/packages/knowPro/src/search.ts b/ts/packages/knowPro/src/search.ts
@@ -141,6 +141,14 @@ export function getDistinctEntityMatches(
     return q.mergeEntityMatches(semanticRefs, searchResults, topK);
 }
 
+export function getDistinctTopicMatches(
+    semanticRefs: SemanticRef[],
+    searchResults: ScoredSemanticRef[],
+    topK?: number,
+) {
+    return q.mergeTopics(semanticRefs, searchResults, topK);
+}
+
 class SearchQueryBuilder {
     // All SearchTerms used which compiling the 'select' portion of the query
     private allSearchTerms: SearchTerm[] = [];