PDF viewer - fix various rendering bugs (#1332)

- Fix mismatch between text position and highlight when in higher zoom
levels
- Fix case where contextual toolbar would close almost immediately after
opening
- Fix initialization error when accessing the pdf viewer URL directly

Author: hillary-mutisya

ts/packages/agents/browser/src/extension/contentScript/pdfInterceptor.ts

@@ -64,7 +64,7 @@ export class PDFInterceptor {
     private async performConnectionCheck(): Promise<boolean> {
         try {
             const response = await chrome.runtime.sendMessage({
-                action: "checkWebSocketConnection",
+                type: "checkWebSocketConnection",
             });
             this.isWebSocketConnected = response?.connected || false;
             debug(`WebSocket connection status: ${this.isWebSocketConnected}`);

ts/packages/agents/browser/src/extension/contentScript/recording/index.ts

@@ -126,7 +126,7 @@ export function observeDOMChanges(): void {
 
             // Optional: Send message to background script
             chrome.runtime.sendMessage({
-                action: "spaNavigationDetected",
+                type: "spaNavigationDetected",
                 url: currentUrl,
             });
 

ts/packages/agents/browser/src/views/client/pdf/app.ts

@@ -284,6 +284,9 @@ export class TypeAgentPDFViewerApp {
     ): void {
         console.log("🎯 Highlight clicked:", highlightId);
 
+        // Prevent selection manager from closing toolbar immediately after we open it
+        this.selectionManager?.ignoreSelectionChangesFor(300);
+
         // Create a fake selection for the highlight area
         const fakeSelection: SelectionInfo = {
             text: highlightData.text || "Highlighted text",
@@ -799,8 +802,15 @@ export class TypeAgentPDFViewerApp {
         const toolbar = document.querySelector(".toolbar") as HTMLElement;
         const toolbarHeight = toolbar ? toolbar.offsetHeight : 56;
 
+        // Set container positioning - PDF.js requires absolute positioning
         viewerContainer.style.cssText = `
-            position: relative; top: 0; left: 0; right: 0; height: calc(100vh - ${toolbarHeight}px); overflow: auto;
+            position: absolute; 
+            top: ${toolbar ? toolbarHeight : 0}px; 
+            left: 0; 
+            right: 0; 
+            bottom: 0; 
+            overflow: auto;
+            background: #323639;
         `;
 
         const linkService = new window.pdfjsViewer.PDFLinkService({

ts/packages/agents/browser/src/views/client/pdf/core/pdfJSHighlightManager.ts

@@ -87,7 +87,7 @@ export class PDFJSHighlightManager {
                 coordinates,
                 pageNumber: selection.pageNumber,
                 text: selection.text,
-                creationScale: this.pdfViewer.currentScale || 1, // Store scale at creation time
+                coordinateScale: coordinates.coordinateScale, // Store the scale at which coordinates were calculated
             };
 
             // Store highlight
@@ -169,7 +169,7 @@ export class PDFJSHighlightManager {
     }
 
     /**
-     * Convert selection to coordinates using PDF.js viewport
+     * Convert selection to coordinates using PDF.js text layer
      */
     private convertSelectionToCoordinates(
         selection: SelectionInfo,
@@ -186,52 +186,73 @@ export class PDFJSHighlightManager {
                 return null;
             }
 
-            const viewport = pageView.viewport;
             const pageElement = pageView.div;
-
             if (!pageElement) {
                 console.error("Page element not found");
                 return null;
             }
 
-            const pageRect = pageElement.getBoundingClientRect();
+            // Find the text layer within the page
+            const textLayer = pageElement.querySelector(".textLayer");
+            if (!textLayer) {
+                console.error(
+                    "Text layer not found for page:",
+                    selection.pageNumber,
+                );
+                return null;
+            }
+
+            const textLayerRect = textLayer.getBoundingClientRect();
+            const currentScale = this.pdfViewer.currentScale || 1;
 
-            // Calculate bounds from selection rectangles relative to page
+            console.log("🔍 Text layer rect:", textLayerRect);
+            console.log("🔍 Current scale:", currentScale);
+
+            // Calculate bounds from selection rectangles relative to text layer
             let minLeft = Infinity;
             let maxRight = -Infinity;
             let minTop = Infinity;
             let maxBottom = -Infinity;
 
             for (const rect of selection.rects) {
-                const relativeLeft = rect.left - pageRect.left;
-                const relativeTop = rect.top - pageRect.top;
-                const relativeRight = rect.right - pageRect.left;
-                const relativeBottom = rect.bottom - pageRect.top;
+                console.log("🔍 Selection rect:", rect);
+
+                // Calculate position relative to text layer
+                const relativeLeft = rect.left - textLayerRect.left;
+                const relativeTop = rect.top - textLayerRect.top;
+                const relativeRight = rect.right - textLayerRect.left;
+                const relativeBottom = rect.bottom - textLayerRect.top;
 
                 minLeft = Math.min(minLeft, relativeLeft);
                 maxRight = Math.max(maxRight, relativeRight);
                 minTop = Math.min(minTop, relativeTop);
                 maxBottom = Math.max(maxBottom, relativeBottom);
             }
 
-            return {
+            const coordinates = {
                 x: minLeft,
                 y: minTop,
                 width: maxRight - minLeft,
                 height: maxBottom - minTop,
                 pageRect: {
-                    width: pageRect.width,
-                    height: pageRect.height,
+                    width: textLayerRect.width,
+                    height: textLayerRect.height,
                 },
+                // Store coordinates are relative to text layer at current scale
+                coordinateScale: currentScale,
+                coordinateSystem: "textLayer", // Mark that these coordinates are relative to text layer
             };
+
+            console.log("🔍 Calculated coordinates:", coordinates);
+            return coordinates;
         } catch (error) {
             console.error("Failed to convert selection to coordinates:", error);
             return null;
         }
     }
 
     /**
-     * Render highlight on the page using DOM overlay with scale-aware positioning
+     * Render highlight on the page using text layer positioning
      */
     private renderHighlight(highlightData: any): void {
         try {
@@ -245,8 +266,15 @@ export class PDFJSHighlightManager {
 
             const pageElement = pageView.div;
 
-            // Create or get highlight layer
-            let highlightLayer = pageElement.querySelector(
+            // Find the text layer within the page
+            const textLayer = pageElement.querySelector(".textLayer");
+            if (!textLayer) {
+                console.error("Text layer not found for rendering highlight");
+                return;
+            }
+
+            // Create or get highlight layer within the text layer
+            let highlightLayer = textLayer.querySelector(
                 ".pdfjs-highlight-layer",
             );
             if (!highlightLayer) {
@@ -261,7 +289,7 @@ export class PDFJSHighlightManager {
                     pointer-events: none;
                     z-index: 1;
                 `;
-                pageElement.appendChild(highlightLayer);
+                textLayer.appendChild(highlightLayer);
             }
 
             // Create highlight element
@@ -272,40 +300,56 @@ export class PDFJSHighlightManager {
                 highlightData.id,
             );
 
-            // Get current scale and calculate scaled coordinates
-            const currentScale = this.pdfViewer.currentScale || 1;
-            const creationScale = highlightData.creationScale || 1;
             const coords = highlightData.coordinates;
+            const currentScale = this.pdfViewer.currentScale || 1;
 
-            // Scale the coordinates: (original coordinates / creation scale) * current scale
-            const scaleRatio = currentScale / creationScale;
-            const scaledCoords = {
-                x: coords.x * scaleRatio,
-                y: coords.y * scaleRatio,
-                width: coords.width * scaleRatio,
-                height: coords.height * scaleRatio,
+            // For text layer coordinates, we don't need to apply additional scaling
+            // since the text layer itself scales with PDF.js
+            let finalCoords = {
+                x: coords.x,
+                y: coords.y,
+                width: coords.width,
+                height: coords.height,
             };
 
+            // Only apply scaling if coordinates were stored with a different coordinate system
+            if (coords.coordinateSystem !== "textLayer") {
+                // Legacy coordinates or coordinates from page-relative system
+                const coordinateScale =
+                    coords.coordinateScale ||
+                    highlightData.coordinateScale ||
+                    1;
+                const scaleRatio = currentScale / coordinateScale;
+                finalCoords = {
+                    x: coords.x * scaleRatio,
+                    y: coords.y * scaleRatio,
+                    width: coords.width * scaleRatio,
+                    height: coords.height * scaleRatio,
+                };
+            }
+
+            console.log("🎨 Rendering highlight with coords:", finalCoords);
+
             highlightElement.style.cssText = `
                 position: absolute;
-                left: ${scaledCoords.x}px;
-                top: ${scaledCoords.y}px;
-                width: ${scaledCoords.width}px;
-                height: ${scaledCoords.height}px;
+                left: ${finalCoords.x}px;
+                top: ${finalCoords.y}px;
+                width: ${finalCoords.width}px;
+                height: ${finalCoords.height}px;
                 background-color: ${highlightData.color};
-                opacity: 0.3;
+                opacity: 0.8;
                 pointer-events: auto;
                 border-radius: 2px;
                 transition: opacity 0.15s ease;
             `;
 
             // Add hover effect
             highlightElement.addEventListener("mouseenter", () => {
-                highlightElement.style.opacity = "0.5";
+                highlightElement.style.opacity = "1";
             });
 
             highlightElement.addEventListener("mouseleave", () => {
-                highlightElement.style.opacity = "0.3";
+                highlightElement.style.opacity = "0.8";
             });
 
             // Add click handler for highlight interaction
@@ -381,9 +425,13 @@ export class PDFJSHighlightManager {
                     y: highlightData.coordinates.y,
                     width: highlightData.coordinates.width,
                     height: highlightData.coordinates.height,
+                    coordinateScale: highlightData.coordinates.coordinateScale,
+                    coordinateSystem:
+                        highlightData.coordinates.coordinateSystem ||
+                        "textLayer",
                 },
                 color: highlightData.color,
-                opacity: 0.3,
+                opacity: 0.8,
                 content: highlightData.text,
                 storage: "pdfjs" as const,
                 createdAt: new Date().toISOString(),
@@ -404,10 +452,17 @@ export class PDFJSHighlightManager {
         return {
             id: apiAnnotation.id,
             color: apiAnnotation.color,
-            coordinates: apiAnnotation.coordinates,
+            coordinates: {
+                ...apiAnnotation.coordinates,
+                // For legacy data without coordinateSystem, assume page-based coordinates
+                coordinateSystem:
+                    apiAnnotation.coordinates?.coordinateSystem || "page",
+                coordinateScale:
+                    apiAnnotation.coordinates?.coordinateScale || 1,
+            },
             pageNumber: apiAnnotation.page,
             text: apiAnnotation.content || "",
-            creationScale: 1, // Assume existing highlights were created at 1x scale
+            coordinateScale: apiAnnotation.coordinates?.coordinateScale || 1,
         };
     }
 

ts/packages/agents/browser/src/views/client/pdf/core/textSelectionManager.ts

@@ -31,12 +31,21 @@ export class TextSelectionManager {
     private selectionTimeout: number | null = null;
     private pdfViewer: any;
     private contextualToolbar: any = null;
+    private ignoreSelectionChangeUntil: number = 0; // Timestamp to ignore selection changes until
 
     constructor(pdfViewer: any) {
         this.pdfViewer = pdfViewer;
         this.setupEventListeners();
     }
 
+    /**
+     * Temporarily ignore selection changes (used when highlight click opens toolbar)
+     */
+    ignoreSelectionChangesFor(durationMs: number = 300): void {
+        this.ignoreSelectionChangeUntil = Date.now() + durationMs;
+        console.log("🔇 Ignoring selection changes for", durationMs, "ms");
+    }
+
     /**
      * Register a callback for selection changes
      */
@@ -180,6 +189,14 @@ export class TextSelectionManager {
      * Process the current selection and update state
      */
     private processSelectionChange(): void {
+        // Check if we should ignore selection changes (e.g., after highlight click)
+        if (Date.now() < this.ignoreSelectionChangeUntil) {
+            console.log(
+                "🔇 Ignoring selection change due to recent highlight click",
+            );
+            return;
+        }
+
         const selection = window.getSelection();
 
         if (!selection || selection.rangeCount === 0) {

ts/packages/agents/browser/src/views/client/pdf/pdf-viewer.css

@@ -154,9 +154,14 @@ body {
   height: 100%;
 }
 
-/* PDF.js viewer container - positioned by JavaScript below toolbar */
+/* PDF.js viewer container - must be absolutely positioned for PDF.js */
 #viewerContainer {
-  /* Position will be set dynamically by JavaScript */
+  position: absolute;
+  top: 0;
+  left: 0;
+  right: 0;
+  bottom: 0;
+  overflow: auto;
 }
 
 #viewer {

ts/packages/agents/browser/src/views/client/pdf/styles/annotation-styles.css

@@ -32,7 +32,7 @@
 }
 
 .pdfjs-highlight:hover {
-  opacity: 0.5 !important;
+  opacity: 1 !important;
 }
 
 /* Custom Highlight Annotations */