feat: added libsql support in embedJs

Author: adhityan

.gitignore

@@ -64,6 +64,7 @@ acme.json
 temp/
 storage/
 lmdb/
+data.db
 
 #env
 .env

core/embedjs/src/core/rag-application.ts

@@ -306,6 +306,17 @@ export class RAGApplication {
         return this.vectorDatabase.getVectorCount();
     }
 
+    /**
+     * The function `deleteConversation` deletes all entries related to a particular conversation from the database
+     * @param {string} conversationId - The `conversationId` that you want to delete. Pass 'default' to delete
+     * the default conversation thread that is created and maintained automatically
+     */
+    public async deleteConversation(conversationId: string) {
+        if (this.store) {
+            await this.store.deleteConversation(conversationId);
+        }
+    }
+
     /**
      * The function `deleteLoader` deletes embeddings from a loader after confirming the action.
      * @param {string} uniqueLoaderId - The `uniqueLoaderId` parameter is a string that represents the

core/embedjs/src/store/memory-store.ts

@@ -50,16 +50,22 @@ export class MemoryStore implements BaseStore {
         const loaderId = <string>this.loaderCustomValues[key].loaderId;
 
         delete this.loaderList[key];
-        this.loaderCustomValuesMap.set(
-            loaderId,
-            this.loaderCustomValuesMap.get(loaderId).filter((k) => k !== key),
-        );
+
+        if (this.loaderCustomValuesMap.has(loaderId)) {
+            this.loaderCustomValuesMap.set(
+                loaderId,
+                this.loaderCustomValuesMap.get(loaderId).filter((k) => k !== key),
+            );
+        }
     }
 
     async deleteLoaderMetadataAndCustomValues(loaderId: string): Promise<void> {
-        this.loaderCustomValuesMap.get(loaderId).forEach((key) => {
-            delete this.loaderCustomValues[key];
-        });
+        if (this.loaderCustomValuesMap.has(loaderId)) {
+            this.loaderCustomValuesMap.get(loaderId).forEach((key) => {
+                delete this.loaderCustomValues[key];
+            });
+        }
+
         this.loaderCustomValuesMap.delete(loaderId);
         delete this.loaderList[loaderId];
     }

databases/embedjs-libsql/package.json

@@ -4,6 +4,7 @@
     "description": "Add LibSQL support to embedjs",
     "dependencies": {
         "@libsql/client": "^0.14.0",
+        "@llm-tools/embedjs-utils": "0.1.21",
         "@llm-tools/embedjs-interfaces": "0.1.21",
         "debug": "^4.3.7"
     },

databases/embedjs-libsql/src/index.ts

@@ -1 +1,2 @@
 export * from './libsql-db.js';
+export * from './libsql-store.js';

databases/embedjs-libsql/src/libsql-db.ts

@@ -2,6 +2,7 @@ import createDebugMessages from 'debug';
 import { Client, createClient } from '@libsql/client';
 
 import { BaseVectorDatabase, ExtractChunkData, InsertChunkData } from '@llm-tools/embedjs-interfaces';
+import { truncateCenterString } from '@llm-tools/embedjs-utils';
 
 export class LibSqlDb implements BaseVectorDatabase {
     private readonly debug = createDebugMessages('embedjs:vector:LibSqlDb');
@@ -16,46 +17,48 @@ export class LibSqlDb implements BaseVectorDatabase {
     }
 
     async init({ dimensions }: { dimensions: number }) {
-        await this.client.execute(`CREATE TABLE ${this.tableName} (
-            id              TEXT
-            pageContent     TEXT,
-            uniqueLoaderId  TEXT,
-            source          TEXT,
+        await this.client.execute(`CREATE TABLE IF NOT EXISTS ${this.tableName} (
+            id              TEXT PRIMARY KEY,
+            pageContent     TEXT UNIQUE,
+            uniqueLoaderId  TEXT NOT NULL,
+            source          TEXT NOT NULL,
             vector          F32_BLOB(${dimensions}),
             metadata        TEXT
         );`);
     }
 
     async insertChunks(chunks: InsertChunkData[]): Promise<number> {
-        const values = chunks.map((chunk) => {
-            return `('${chunk.metadata.id}', '${chunk.pageContent}', '${chunk.metadata.uniqueLoaderId}', 
-                '${chunk.metadata.source}', vector32('[${chunk.vector.join(',')}]'),
-                '${JSON.stringify(chunk.metadata)}')`;
+        const batch = chunks.map((chunk) => {
+            return {
+                sql: `INSERT OR IGNORE INTO ${this.tableName} (id, pageContent, uniqueLoaderId, source, vector, metadata)
+            VALUES (?, ?, ?, ?, vector32('[${chunk.vector.join(',')}]'), ?);`,
+                args: [
+                    chunk.metadata.id,
+                    chunk.pageContent,
+                    chunk.metadata.uniqueLoaderId,
+                    chunk.metadata.source,
+                    JSON.stringify(chunk.metadata),
+                ],
+            };
         });
 
-        const statement = `INSERT INTO ${this.tableName} (id, pageContent, uniqueLoaderId, source, vector, metadata)
-            VALUES
-            ${values.join(',')}
-        );`;
-
-        this.debug(`Executing statement - ${statement}`);
-        const result = await this.client.execute(statement);
-        return result.rowsAffected;
+        this.debug(`Executing batch - ${truncateCenterString(JSON.stringify(batch), 1000)}`);
+        const result = await this.client.batch(batch, 'write');
+        return result.reduce((a, b) => a + b.rowsAffected, 0);
     }
 
     async similaritySearch(query: number[], k: number): Promise<ExtractChunkData[]> {
         const statement = `SELECT id, pageContent, uniqueLoaderId, source, metadata,
                 vector_distance_cos(vector, vector32('[${query.join(',')}]'))
             FROM ${this.tableName}
-            ORDER BY vector_distance_cos(vector, vector32('[${query.join(',')}]'))
-            TOP ${k}
-            ASC;`;
+            ORDER BY vector_distance_cos(vector, vector32('[${query.join(',')}]')) ASC
+            LIMIT ${k};`;
 
-        this.debug(`Executing statement - ${statement}`);
+        this.debug(`Executing statement - ${truncateCenterString(statement, 700)}`);
         const results = await this.client.execute(statement);
 
         return results.rows.map((result) => {
-            const metadata = JSON.parse(<string>result.metadata.toString());
+            const metadata = JSON.parse(result.metadata.toString());
 
             return {
                 metadata,
@@ -66,16 +69,21 @@ export class LibSqlDb implements BaseVectorDatabase {
     }
 
     async getVectorCount(): Promise<number> {
-        return 0;
+        const statement = `SELECT count(id) as count FROM ${this.tableName};`;
+
+        this.debug(`Executing statement - ${statement}`);
+        const results = await this.client.execute(statement);
+
+        return Number.parseInt(results.rows[0].count.toString());
     }
 
     async deleteKeys(uniqueLoaderId: string): Promise<boolean> {
-        const statement = `DELETE FROM ${this.tableName} VALUES WHERE
-           uniqueLoaderId = '${uniqueLoaderId}';`;
-
-        await this.client.execute(statement);
+        await this.client.execute(`DELETE FROM ${this.tableName} WHERE
+           uniqueLoaderId = '${uniqueLoaderId}';`);
         return true;
     }
 
-    async reset(): Promise<void> {}
+    async reset(): Promise<void> {
+        await this.client.execute(`DELETE FROM ${this.tableName};`);
+    }
 }