Add searchindex to mockserver

e2e/cypress/integration/local-index.ts

@@ -0,0 +1,27 @@
+/// <reference types="cypress" />
+
+describe('Using the local index', () => {
+    beforeEach(() => {
+        localStorage.setItem('localSearchPromptDisplayed221', 'true');
+    });
+
+    it('can download and remove the local index', () => {
+        cy.visit('/');
+        const selector = 'mat-nav-list:nth-of-type(2) mat-list-item';
+
+        cy.server();
+        cy.route('POST', '**/verifymessages').as('messagesverified');
+
+        cy.get(selector).should('contain', 'Synchronize index').click()
+        cy.get(selector).should('contain', 'Stop index synchronization');
+
+        cy.wait('@messagesverified');
+
+        cy.get('#searchField input').type('default fix');
+        cy.get('#searchField input').invoke('attr', 'placeholder').should('contain', 'showing 1 hit');
+
+        cy.visit('/');
+        cy.get(selector).should('contain', 'Stop index synchronization').click();
+        cy.get(selector).should('contain', 'Synchronize index');
+    });
+})

e2e/mockserver/mockserver.ts

@@ -20,6 +20,12 @@
 import { createServer, Server } from 'http';
 import { createWriteStream } from 'fs';
 import { mail_message_obj } from './emailresponse';
+import { XapianAPI } from 'runbox-searchindex/rmmxapianapi';
+import { loadXapian } from 'runbox-searchindex/xapian.loader';
+import { IndexingTools, MessageInfo } from 'runbox-searchindex/messageinfo';
+import { MailAddressInfo } from 'runbox-searchindex/mailaddressinfo';
+
+declare var FS: any, MEMFS: any;
 
 const logger = createWriteStream('mockserver.log');
 function log(line) {
@@ -39,6 +45,7 @@ export class MockServer {
     ];
 
     events = [];
+    indexPartitions = [];
 
     folders = [
         {
@@ -107,6 +114,10 @@ export class MockServer {
         },
     ];
 
+    constructor() {
+        loadXapian(`${process.cwd()}/node_modules/runbox-searchindex/xapianasm.js`).subscribe(() => this.buildSearchIndex());
+    }
+
     public start() {
         log('Starting mock server');
         this.server = createServer((request, response) => {
@@ -140,9 +151,18 @@ export class MockServer {
                 } else if (requesturl.indexOf('folder=Inbox') > -1) {
                     requesturl = '/mail/download_xapian_index?inbox';
                 } else {
-                    requesturl = '/mail/download_xapian_index';
+                    const match = requesturl.match(/fileno=(\d+)/);
+                    if (match) {
+                        const files = ["iamglass","docdata.glass","termlist.glass","postlist.glass"];
+                        const fileno = parseInt(match[1], 10);
+                        const path = './downloadable/' + files[fileno - 1];
+                        console.log(`Uploading ${path}`);
+                        const stat = FS.stat(path);
+                        const bytes = FS.readFile(path, { encoding: 'binary' }, stat.size);
+                        response.end(Buffer.from(bytes));
+                        return;
+                    }
                 }
-
             }
             const emailendpoint = requesturl.match(/\/rest\/v1\/email\/([0-9]+)/);
             if (emailendpoint) {
@@ -260,6 +280,9 @@ export class MockServer {
                 case '/mail/download_xapian_index':
                     response.end('');
                     break;
+                case '/mail/download_xapian_index?exists=check':
+                    response.end(JSON.stringify({ 'exists': true }));
+                    break;
                 case '/mail/download_xapian_index?inbox':
                     response.end(this.inboxcontents());
                     break;
@@ -281,6 +304,9 @@ export class MockServer {
                         }
                     ));
                     break;
+                case '/rest/v1/searchindex/partitions':
+                    response.end(JSON.stringify({ "partitions": this.indexPartitions }));
+                    break;
                 default:
                     if (request.url.indexOf('/rest') === 0) {
                         response.end(JSON.stringify({ status: 'success' }));
@@ -691,4 +717,71 @@ export class MockServer {
             ],
         ];
     }
+
+    buildSearchIndex() {
+        FS.mkdir("/work");
+        FS.mount(MEMFS, {},"/work");
+        FS.chdir("/work");
+        const xapian = new XapianAPI();
+        xapian.initXapianIndex('mainpartition');
+        const indexer = new IndexingTools(xapian);
+
+        const lines = this.inboxcontents().split('\n');
+        // copied from rbwebmail.ts -- refactor!
+        lines.map((line) => {
+            const parts = line.split('\t');
+            const from_ = parts[7];
+            const to = parts[8];
+            const fromInfo: MailAddressInfo[] = MailAddressInfo.parse(from_);
+            const toInfo: MailAddressInfo[] = MailAddressInfo.parse(to);
+            const size: number = parseInt(parts[10], 10);
+            const attachment: boolean = parts[11] === 'y';
+            const seenFlag: boolean = parseInt(parts[4], 10) === 1;
+            const answeredFlag: boolean = parseInt(parts[5], 10) === 1;
+            const flaggedFlag: boolean = parseInt(parts[6], 10) === 1;
+
+
+            const ret = new MessageInfo(
+                parseInt(parts[0], 10), // id
+                new Date(), // changed date
+                new Date(), // message date
+                parts[3],                                // folder
+                seenFlag,                                // seen flag
+                answeredFlag,                            // answered flag
+                flaggedFlag,                             // flagged flag
+                fromInfo,                                // from
+                toInfo,                                  // to
+                [],                                      // cc
+                [],                                      // bcc
+                parts[9],                                // subject
+                parts[12],                               // plaintext body
+                size,                                    // size
+                attachment                               // attachment
+            );
+            if (size === -1) {
+                // Size = -1 means deleted flag is set - ref hack in Webmail.pm
+                ret.deletedFlag = true;
+            }
+            return ret;
+        }).forEach(msg => indexer.addMessageToIndex(msg));
+        const docCount = xapian.getXapianDocCount();
+        console.log("Stored", docCount, "documents in xapian database");
+        xapian.commitXapianUpdates();
+        xapian.compactToWritableDatabase('downloadable');
+        xapian.closeXapianDatabase();
+        this.indexPartitions = [{
+            'folder': 'downloadable',
+            files: [],
+            numberOfMessages: docCount,
+        }]
+        FS.readdir('./downloadable').forEach(file => {
+            const path = './downloadable/' + file;
+            const size = FS.stat(path).size;
+            this.indexPartitions[0].files.push({
+                'filename': file,
+                'compressedsize': size,
+                'uncompressedsize': size,
+            });
+        });
+    }
 }

package.json

@@ -56,7 +56,7 @@
     "moment-timezone": "^0.5.28",
     "npm": "^6.14.8",
     "rrule": "^2.6.4",
-    "runbox-searchindex": "https://registry.npmjs.org/@runboxcom/runbox-searchindex/-/runbox-searchindex-0.2.0.tgz",
+    "runbox-searchindex": "https://registry.npmjs.org/@runboxcom/runbox-searchindex/-/runbox-searchindex-0.2.1.tgz",
     "rxjs": "^6.5.5",
     "rxjs-compat": "^6.5.5",
     "tinymce": "^5.2.2",

src/app/xapian/searchservice.ts

@@ -193,8 +193,6 @@ export class SearchService {
           this.updateIndexWithNewChanges();
           this.noLocalIndexFoundSubject.next(true);
           this.noLocalIndexFoundSubject.complete();
-          this.initSubject.next(false);
-          this.initSubject.complete();
         }
       });
   }
@@ -486,6 +484,7 @@ export class SearchService {
           mergeMap(() => downloadAndWriteFile('postlist.glass', 4)),
         ).subscribe(() => {
             this.api.initXapianIndex(XAPIAN_GLASS_WR);
+            this.indexNotPersisted = true;
             console.log(this.api.getXapianDocCount() + ' docs in Xapian database');
             this.localSearchActivated = true;
             this.messagelistservice.refreshFolderCounts();