WebMemories: filter html page to the core readable view before running entity and topic extraction (#1339)

Author: hillary-mutisya

ts/packages/agents/browser/src/agent/knowledge/knowledgeHandler.mts

@@ -52,6 +52,25 @@ interface AnalyticsDataResponse {
         totalActions: number;
         totalRelationships: number;
         recentItems?: any[];
+        recentEntities?: Array<{
+            name: string;
+            type: string;
+            fromPage: string;
+            extractedAt: string;
+        }>;
+        recentTopics?: Array<{
+            name: string;
+            fromPage: string;
+            extractedAt: string;
+        }>;
+        recentActions?: Array<{
+            type: string;
+            element: string;
+            text?: string;
+            confidence: number;
+            fromPage: string;
+            extractedAt: string;
+        }>;
     };
     domains: {
         topDomains: Array<{
@@ -487,11 +506,33 @@ export async function indexWebPageContent(
                         : [entity.type], // Ensure type is array
                 })),
                 topics: aggregatedResults.keyTopics,
-                actions: [], // Actions would need to be extracted separately if needed
+                actions: aggregatedResults.contentActions || [], // Use actual content actions
                 inverseActions: [], // Required property
             };
         }
 
+        // Store detectedActions and actionSummary in metadata for retrieval
+        if (
+            aggregatedResults &&
+            (aggregatedResults.detectedActions ||
+                aggregatedResults.actionSummary)
+        ) {
+            websiteObj.metadata = websiteObj.metadata || {};
+
+            if (
+                aggregatedResults.detectedActions &&
+                aggregatedResults.detectedActions.length > 0
+            ) {
+                websiteObj.metadata.detectedActions =
+                    aggregatedResults.detectedActions;
+            }
+
+            if (aggregatedResults.actionSummary) {
+                websiteObj.metadata.actionSummary =
+                    aggregatedResults.actionSummary;
+            }
+        }
+
         if (context.agentContext.websiteCollection) {
             if (parameters.extractKnowledge) {
                 try {
@@ -1695,11 +1736,23 @@ export async function getPageIndexedKnowledge(
             }
 
             let detectedActions: any[] = [];
+
+            // Check websiteObj metadata for detectedActions first (with safe property access)
+            if (
+                foundWebsite.metadata &&
+                (foundWebsite.metadata as any).detectedActions &&
+                Array.isArray((foundWebsite.metadata as any).detectedActions)
+            ) {
+                detectedActions = (foundWebsite.metadata as any)
+                    .detectedActions;
+            }
+
+            // Also check knowledge object for detectedActions (fallback)
             if (
                 (knowledge as any).detectedActions &&
                 Array.isArray((knowledge as any).detectedActions)
             ) {
-                detectedActions.push(...(knowledge as any).detectedAction);
+                detectedActions.push(...(knowledge as any).detectedActions);
             }
 
             // Convert the stored knowledge to the expected format
@@ -1762,6 +1815,10 @@ export async function getPageIndexedKnowledge(
                     relationships,
                     keyTopics,
                     detectedActions,
+                    contentActions: knowledge.actions || [],
+                    actionSummary: foundWebsite.metadata
+                        ? (foundWebsite.metadata as any).actionSummary
+                        : undefined,
                     suggestedQuestions,
                     summary,
                     contentMetrics,
@@ -2503,6 +2560,7 @@ export async function getAnalyticsData(
             topDomains,
             activityTrends,
             extractionAnalytics,
+            recentKnowledgeItems,
         ] = await Promise.all([
             getDetailedKnowledgeStats(
                 {
@@ -2531,6 +2589,7 @@ export async function getAnalyticsData(
                 },
                 context,
             ),
+            getRecentKnowledgeItems({ limit: 10, type: "all" }, context),
         ]);
 
         // Get basic website statistics from websiteCollection
@@ -2578,6 +2637,9 @@ export async function getAnalyticsData(
                 totalActions: 0, // Actions not tracked in current schema
                 totalRelationships: knowledgeStats.totalRelationships || 0,
                 recentItems: knowledgeStats.recentActivity || [],
+                recentEntities: recentKnowledgeItems.entities || [],
+                recentTopics: recentKnowledgeItems.topics || [],
+                recentActions: recentKnowledgeItems.actions || [],
             },
             domains: {
                 topDomains: topDomains.domains || [],

ts/packages/agents/browser/src/extension/contentScript/eventHandlers.ts

@@ -212,6 +212,8 @@ export async function handleMessage(
                     message.inputHtml,
                     message.frameId,
                     message.useTimestampIds,
+                    message.filterToReadingView,
+                    message.keepMetaTags,
                 );
                 sendResponse(html);
                 break;

ts/packages/agents/browser/src/extension/contentScript/htmlProcessing.ts

@@ -5,20 +5,25 @@ import { HTMLReducer } from "./htmlReducer";
 import DOMPurify from "dompurify";
 import { setIdsOnAllElements, markInvisibleNodesForCleanup } from "./domUtils";
 import { HtmlFragment } from "./types";
+import { Readability, isProbablyReaderable } from "@mozilla/readability";
 
 /**
  * Gets the HTML of the page
  * @param fullSize Whether to get the full HTML
  * @param documentHtml The HTML to process
  * @param frameId The frame ID
  * @param useTimestampIds Whether to use timestamp IDs
+ * @param filterToReadingView Whether to apply readability filter
+ * @param keepMetaTags Whether to preserve meta tags when using readability
  * @returns The processed HTML
  */
 export function getPageHTML(
     fullSize?: boolean,
     documentHtml?: string,
     frameId?: number,
     useTimestampIds?: boolean,
+    filterToReadingView?: boolean,
+    keepMetaTags?: boolean,
 ): string {
     if (!documentHtml) {
         if (frameId !== undefined) {
@@ -28,12 +33,23 @@ export function getPageHTML(
         documentHtml = document.children[0].outerHTML;
     }
 
+    // Apply Readability filter if requested
+    if (filterToReadingView) {
+        documentHtml = applyReadabilityFilter(documentHtml, keepMetaTags);
+    }
+
     if (fullSize) {
         return documentHtml;
     }
 
     const reducer = new HTMLReducer();
     reducer.removeDivs = false;
+
+    // Preserve meta tags if requested and readability was used
+    if (filterToReadingView && keepMetaTags) {
+        reducer.removeMetaTags = false;
+    }
+
     const reducedHtml = reducer.reduce(documentHtml);
     return reducedHtml;
 }
@@ -137,3 +153,59 @@ export function getPageHTMLFragments(
 
     return htmlFragments;
 }
+
+/**
+ * Apply Readability filter to extract main content
+ * @param html The HTML to process
+ * @param keepMetaTags Whether to preserve meta tags
+ * @returns The processed HTML with main content extracted
+ */
+function applyReadabilityFilter(html: string, keepMetaTags?: boolean): string {
+    try {
+        // Parse the HTML
+        const domParser = new DOMParser();
+        const doc = domParser.parseFromString(html, "text/html");
+
+        // Check if readability can process this document
+        if (!isProbablyReaderable(doc)) {
+            console.warn(
+                "Document is not probably readerable, skipping Readability filter",
+            );
+            return html;
+        }
+
+        // Clone document to avoid modifying original
+        const documentClone = doc.cloneNode(true) as Document;
+
+        // Extract meta tags before applying Readability if we want to preserve them
+        let metaTags = "";
+        if (keepMetaTags) {
+            const headClone = documentClone.head?.cloneNode(
+                true,
+            ) as HTMLHeadElement;
+            if (headClone) {
+                const metaElements = headClone.querySelectorAll("meta, title");
+                metaTags = Array.from(metaElements)
+                    .map((el) => el.outerHTML)
+                    .join("\n");
+            }
+        }
+
+        // Apply Readability
+        const article = new Readability(documentClone).parse();
+
+        if (article?.content) {
+            // Construct new HTML with main content
+            let resultHtml = `<html><head>${metaTags}</head><body>${article.content}</body></html>`;
+            return resultHtml;
+        } else {
+            console.warn(
+                "Readability failed to extract content, falling back to original HTML",
+            );
+            return html;
+        }
+    } catch (error) {
+        console.error("Error applying Readability filter:", error);
+        return html; // Fallback to original HTML
+    }
+}

ts/packages/agents/browser/src/extension/serviceWorker/browserActions.ts

@@ -181,6 +181,8 @@ export async function runBrowserAction(action: AppAction): Promise<any> {
                 action.parameters?.downloadAsFile,
                 action.parameters?.extractText,
                 action.parameters?.useTimestampIds,
+                action.parameters?.filterToReadingView,
+                action.parameters?.keepMetaTags,
             );
             break;
         }

ts/packages/agents/browser/src/extension/serviceWorker/capture.ts

@@ -15,6 +15,8 @@ import {
  * @param downloadAsFile Whether to download the HTML as a file
  * @param extractText Whether to extract text from the HTML
  * @param useTimestampIds Whether to use timestamp IDs
+ * @param filterToReadingView Whether to apply readability filter
+ * @param keepMetaTags Whether to preserve meta tags when using readability
  * @returns Promise resolving to an array of HTML fragments
  */
 export async function getTabHTMLFragments(
@@ -23,6 +25,8 @@ export async function getTabHTMLFragments(
     downloadAsFile?: boolean,
     extractText?: boolean,
     useTimestampIds?: boolean,
+    filterToReadingView?: boolean,
+    keepMetaTags?: boolean,
 ): Promise<HTMLFragment[]> {
     const frames = await chrome.webNavigation.getAllFrames({
         tabId: targetTab.id!,
@@ -41,6 +45,8 @@ export async function getTabHTMLFragments(
                         fullSize: fullSize,
                         frameId: frames[i].frameId,
                         useTimestampIds: useTimestampIds,
+                        filterToReadingView: filterToReadingView,
+                        keepMetaTags: keepMetaTags,
                     },
                     { frameId: frames[i].frameId },
                 );