feat: web search (#57)

Author: nomagick

backend/.gitignore

@@ -75,4 +75,5 @@ build/
 .DS_Store
 
 *.local
-.secret.*
+.secret.*
+licensed/

backend/functions/integrity-check.cjs

@@ -0,0 +1,11 @@
+#!/usr/bin/env node
+
+const fs = require('fs');
+const path = require('path');
+
+const file = path.resolve(__dirname, 'licensed/GeoLite2-City.mmdb');
+
+if (!fs.existsSync(file)) {
+    console.error(`Integrity check failed: ${file} does not exist.`);
+    process.exit(1);
+}

backend/functions/package-lock.json

@@ -2,7 +2,7 @@
   "name": "reader",
   "scripts": {
     "lint": "eslint --ext .js,.ts .",
-    "build": "tsc -p .",
+    "build": "node ./integrity-check.cjs && tsc -p .",
     "build:watch": "tsc --watch",
     "build:clean": "rm -rf ./build",
     "shell": "npm run build && firebase functions:shell",
@@ -44,6 +44,7 @@
     "htmlparser2": "^9.0.0",
     "jose": "^5.1.0",
     "langdetect": "^0.2.1",
+    "maxmind": "^4.3.18",
     "minio": "^7.1.3",
     "openai": "^4.20.0",
     "puppeteer": "^22.7.1",

backend/functions/package.json

@@ -2,7 +2,7 @@ import {
     assignTransferProtocolMeta, marshalErrorLike,
     RPCHost, RPCReflection,
     HashManager,
-    AssertionFailureError, ParamValidationError,
+    AssertionFailureError, ParamValidationError, Defer,
 } from 'civkit';
 import { singleton } from 'tsyringe';
 import { AsyncContext, CloudHTTPv2, Ctx, FirebaseStorageBucketControl, InsufficientBalanceError, Logger, OutputServerEventStream, RPCReflect } from '../shared';
@@ -34,6 +34,12 @@ export class CrawlerHost extends RPCHost {
     cacheValidMs = 1000 * 300;
     urlValidMs = 1000 * 3600 * 4;
 
+    indexText = `[Usage1] https://r.jina.ai/YOUR_URL
+[Usage2] https://s.jina.ai/YOUR_SEARCH_QUERY
+[Homepage] https://jina.ai/reader
+[Source code] https://github.com/jina-ai/reader
+`;
+
     constructor(
         protected globalLogger: Logger,
         protected puppeteerControl: PuppeteerControl,
@@ -357,10 +363,7 @@ ${this.content}
 [Balance left] ${latestUser.wallet.total_balance}
 ` : '';
 
-            return assignTransferProtocolMeta(`[Usage] https://r.jina.ai/YOUR_URL
-[Homepage] https://jina.ai/reader
-[Source code] https://github.com/jina-ai/reader
-${authMixin}`,
+            return assignTransferProtocolMeta(`${this.indexText}${authMixin}`,
                 { contentType: 'text/plain', envelope: null }
             );
         }
@@ -638,13 +641,13 @@ ${authMixin}`,
         return r;
     }
 
-    async *cachedScrap(urlToCrawl: URL, crawlOpts: ScrappingOptions, noCache: boolean = false) {
+    async *cachedScrap(urlToCrawl: URL, crawlOpts?: ScrappingOptions, noCache: boolean = false) {
         let cache;
-        if (!noCache && !crawlOpts.cookies?.length) {
+        if (!noCache && !crawlOpts?.cookies?.length) {
             cache = await this.queryCache(urlToCrawl);
         }
 
-        if (cache?.isFresh && (!crawlOpts.favorScreenshot || (crawlOpts.favorScreenshot && cache?.screenshotAvailable))) {
+        if (cache?.isFresh && (!crawlOpts?.favorScreenshot || (crawlOpts?.favorScreenshot && cache?.screenshotAvailable))) {
             yield cache.snapshot;
 
             return;
@@ -683,4 +686,47 @@ ${authMixin}`,
         return undefined;
     }
 
+
+    async *scrapMany(urls: URL[], options?: ScrappingOptions, noCache = false) {
+        const iterators = urls.map((url) => this.cachedScrap(url, options, noCache));
+
+        const results: (PageSnapshot | undefined)[] = iterators.map((_x)=> undefined);
+
+        let nextDeferred = Defer();
+        let concluded = false;
+
+        const handler = async (it: AsyncGenerator<PageSnapshot | undefined>, idx: number) => {
+            for await (const x of it) {
+                results[idx] = x;
+
+                if (x) {
+                    nextDeferred.resolve();
+                    nextDeferred = Defer();
+                }
+
+            }
+        };
+
+        Promise.all(
+            iterators.map((it, idx) => handler(it, idx))
+        ).finally(() => {
+            concluded = true;
+            nextDeferred.resolve();
+        });
+
+        yield results;
+
+        try {
+            while (!concluded) {
+                await nextDeferred.promise;
+
+                yield results;
+            }
+        } finally {
+            for (const x of iterators) {
+                x.return();
+            }
+        }
+    }
+
 }