fix: cancel timed out requests (#65)

* return dataset it on request timeout

* Handle timed out responses and cancel requests

* Add BoundedArray class to manage timed out responses

* Remove BoundedArray and refactor timeout handling to use responseData

* Remove unused constant TIMED_OUT_RESPONSE_ARRAY_SIZE

* Update CHANGELOG for version 1.0.13

Author: MQ37

Diff for: CHANGELOG.md

@@ -1,5 +1,10 @@
 This changelog summarizes all changes of the RAG Web Browser
 
+### 1.0.13 (2025-03-27)
+
+🐛 Bug Fixes
+- Cancel crawling requests from timed-out search queries
+
 ### 1.0.12 (2025-03-24)
 
 🐛 Bug Fixes

Diff for: src/request-handler.ts

@@ -3,7 +3,7 @@ import { load } from 'cheerio';
 import { CheerioCrawlingContext, htmlToText, log, PlaywrightCrawlingContext, sleep, Request } from 'crawlee';
 
 import { ContentCrawlerStatus, ContentCrawlerTypes } from './const.js';
-import { addResultToResponse, sendResponseIfFinished } from './responses.js';
+import { addResultToResponse, responseData, sendResponseIfFinished } from './responses.js';
 import { Output, ContentCrawlerUserData } from './types.js';
 import { addTimeMeasureEvent, transformTimeMeasuresToRelative } from './utils.js';
 import { processHtml } from './website-content-crawler/html-processing.js';
@@ -27,6 +27,22 @@ async function waitForPlaywright({ page }: PlaywrightCrawlingContext, time: numb
     return Promise.race([page.waitForLoadState('networkidle', { timeout: 0 }), sleep(time - hardDelay)]);
 }
 
+/**
+ * Checks if the request should time out based on response timeout.
+ * It verifies if the response data contains the responseId. If not, it sets the request's noRetry flag
+ * to true and throws an error to cancel the request.
+ *
+ * @param {Request} request - The request object to be checked.
+ * @param {string} responseId - The response ID to look for in the response data.
+ * @throws {Error} Throws an error if the request times out.
+ */
+function checkTimeoutAndCancelRequest(request: Request, responseId: string) {
+    if (!responseData.has(responseId)) {
+        request.noRetry = true;
+        throw new Error('Timed out. Cancelling the request...');
+    }
+}
+
 /**
  * Decide whether to wait based on the remaining time left for the Actor to run.
  * Always waits if the Actor is in the STANDBY_MODE.
@@ -148,7 +164,9 @@ export async function requestHandlerPlaywright(
     context: PlaywrightCrawlingContext<ContentCrawlerUserData>,
 ) {
     const { request, response, page, closeCookieModals } = context;
-    const { contentScraperSettings: settings } = request.userData;
+    const { contentScraperSettings: settings, responseId } = request.userData;
+
+    checkTimeoutAndCancelRequest(request, responseId);
 
     log.info(`Processing URL: ${request.url}`);
     addTimeMeasureEvent(request.userData, 'playwright-request-start');
@@ -180,6 +198,9 @@ export async function requestHandlerCheerio(
     context: CheerioCrawlingContext<ContentCrawlerUserData>,
 ) {
     const { $, request, response } = context;
+    const { responseId } = request.userData;
+
+    checkTimeoutAndCancelRequest(request, responseId);
 
     log.info(`Processing URL: ${request.url}`);
     addTimeMeasureEvent(request.userData, 'cheerio-request-start');

Diff for: src/responses.ts

@@ -11,7 +11,7 @@ type ResponseData = {
     timeoutId?: NodeJS.Timeout;
 };
 
-const responseData = new Map<string, ResponseData>();
+export const responseData = new Map<string, ResponseData>();
 
 /**
  * Helper function to get response object by responseId.
@@ -39,7 +39,7 @@ export function createResponsePromise(responseId: string, timeoutSecs: number):
 
         // Set a timeout to reject the promise if it takes too long
         data.timeoutId = setTimeout(() => {
-            sendResponseError(responseId, 'Timed out');
+            sendResponseError(responseId, 'Timed out.');
         }, timeoutSecs * 1000);
     });
 }