feat: add waitForSelectorTimeout config option

Author: iperzic

config.ts

@@ -18,6 +18,8 @@ type Config = {
     page: Page;
     pushData: (data: any) => Promise<void>;
   }) => Promise<void>;
+    /** Optional timeout for waiting for a selector to appear */
+  waitForSelectorTimeout?: number;
 };
 
 export const config: Config = {

src/main.ts

@@ -33,7 +33,7 @@ if (process.env.NO_CRAWL !== "true") {
       log.info(`Crawling ${request.loadedUrl}...`);
 
       await page.waitForSelector(config.selector, {
-        timeout: 1000,
+        timeout: config.waitForSelectorTimeout ?? 1000,
       });
 
       const html = await getPageHtml(page);