diff --git a/package-lock.json b/package-lock.json index 54e43d3..a0c9ad7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,11 +9,13 @@ "version": "1.0.8", "license": "MIT", "dependencies": { - "@memvid/sdk": "^2.0.146" + "@memvid/sdk": "^2.0.146", + "proper-lockfile": "^4.1.2" }, "devDependencies": { "@eslint/js": "^9.39.2", "@types/node": "^22.0.0", + "@types/proper-lockfile": "^4.1.4", "eslint": "^9.0.0", "tsup": "^8.0.0", "typescript": "^5.7.0", @@ -157,7 +159,8 @@ "version": "4.1.1", "resolved": "https://registry.npmjs.org/@cfworker/json-schema/-/json-schema-4.1.1.tgz", "integrity": "sha512-gAmrUZSGtKc3AiBL71iNWxDsyUC5uMaKKGdvzYsBoTW/xi42JQHl7eKV2OYzCUqvc+D2RCcf7EXY2iCyFIk6og==", - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/@esbuild/aix-ppc64": { "version": "0.27.2", @@ -967,7 +970,6 @@ "version": "0.6.22", "resolved": "https://registry.npmjs.org/@llamaindex/core/-/core-0.6.22.tgz", "integrity": "sha512-/BXyemkvpxMaUhOkbwJ2PTvzKjSWkL8+6QLpz/n+pk8xBwMMe1GVBgli/J57gCyi8GbrlBafBj6GaPOgWub2Eg==", - "peer": true, "dependencies": { "@finom/zod-to-json-schema": "3.24.11", "@llamaindex/env": "0.1.30", @@ -1006,7 +1008,6 @@ "resolved": "https://registry.npmjs.org/zod/-/zod-4.3.5.tgz", "integrity": "sha512-k7Nwx6vuWx1IJ9Bjuf4Zt1PEllcwe7cls3VNzm4CQ1/hgtFUK2bRNG3rvnpPUhFjmqJKAKtjV576KnUkHocg/g==", "license": "MIT", - "peer": true, "funding": { "url": "https://github.com/sponsors/colinhacks" } @@ -1015,7 +1016,6 @@ "version": "0.1.30", "resolved": "https://registry.npmjs.org/@llamaindex/env/-/env-0.1.30.tgz", "integrity": "sha512-y6kutMcCevzbmexUgz+HXf7KiZemzAoFEYSjAILfR+cG6FmYSF8XvLbGOB34Kx8mlRi7EI8rZXpezJ5qCqOyZg==", - "peer": true, "dependencies": { "@aws-crypto/sha256-js": "^5.2.0", "js-tiktoken": "^1.0.12", @@ -1132,7 +1132,8 @@ "resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.2.tgz", "integrity": "sha512-XISRgDJ2Tc5q4TRqvgJtzsRkFYNJzZrhTdtMoGVBttwzzQJkPnS3WWTFc7kuDRoPtPakl+T+OfdEUjYJj7Jbow==", "license": "MIT", - "optional": true + "optional": true, + "peer": true }, "node_modules/@llamaindex/workflow/node_modules/p-retry": { "version": "6.2.1", @@ -1727,11 +1728,20 @@ "integrity": "sha512-1N9SBnWYOJTrNZCdh/yJE+t910Y128BoyY+zBLWhL3r0TYzlTmFdXrPwHL9DyFZmlEXNQQolTZh3KHV31QDhyA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "undici-types": "~6.21.0" } }, + "node_modules/@types/proper-lockfile": { + "version": "4.1.4", + "resolved": "https://registry.npmjs.org/@types/proper-lockfile/-/proper-lockfile-4.1.4.tgz", + "integrity": "sha512-uo2ABllncSqg9F1D4nugVl9v93RmjxF6LJzQLMLDdPaXCUIDPeOJ21Gbqi43xNKzBi/WQ0Q0dICqufzQbMjipQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/retry": "*" + } + }, "node_modules/@types/retry": { "version": "0.12.0", "resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.0.tgz", @@ -1789,7 +1799,6 @@ "integrity": "sha512-iIACsx8pxRnguSYhHiMn2PvhvfpopO9FXHyn1mG5txZIsAaB6F0KwbFnUQN3KCiG3Jcuad/Cao2FAs1Wp7vAyg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "8.52.0", "@typescript-eslint/types": "8.52.0", @@ -2145,7 +2154,6 @@ "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", "dev": true, "license": "MIT", - "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -2232,6 +2240,7 @@ "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", "license": "MIT", + "peer": true, "engines": { "node": ">=10" }, @@ -2342,6 +2351,7 @@ "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz", "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==", "license": "MIT", + "peer": true, "engines": { "node": ">=10" }, @@ -2522,6 +2532,7 @@ "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", "integrity": "sha512-z2S+W9X73hAUUki+N+9Za2lBlun89zigOyGrsax+KUQ6wKW4ZoWpEYBkGhQjwAjjDCkWxhY0VKEhk8wzY7F5cA==", "license": "MIT", + "peer": true, "engines": { "node": ">=0.10.0" } @@ -2633,7 +2644,6 @@ "dev": true, "hasInstallScript": true, "license": "MIT", - "peer": true, "bin": { "esbuild": "bin/esbuild" }, @@ -2688,7 +2698,6 @@ "integrity": "sha512-LEyamqS7W5HB3ujJyvi0HQK/dtVINZvd5mAAp9eT5S/ujByGjiZLCzPcHVzuXbpJDJF/cxwHlfceVUDZ2lnSTw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.1", @@ -3015,6 +3024,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "license": "ISC" + }, "node_modules/has-flag": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", @@ -3125,6 +3140,7 @@ "integrity": "sha512-6oIwpsgRfnDiyEDLMay/GqCl3HoAtH5+RUKW29gYkL0QA+ipzpDLA16yQs7/RHCSu+BwgbJaOUqa4A99qNVQVw==", "license": "MIT", "optional": true, + "peer": true, "engines": { "node": ">=16" }, @@ -3452,6 +3468,7 @@ "resolved": "https://registry.npmjs.org/mustache/-/mustache-4.2.0.tgz", "integrity": "sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ==", "license": "MIT", + "peer": true, "bin": { "mustache": "bin/mustache" } @@ -3498,6 +3515,7 @@ "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.5.0.tgz", "integrity": "sha512-/bRZty2mXUIFY/xU5HLvveNHlswNJej+RnxBjOMkidWfwZzgTbPG1E3K5TOxRLOR+5hX7bSofy8yf1hZevMS8A==", "license": "MIT", + "peer": true, "engines": { "node": "^18 || ^20 || >= 21" } @@ -3507,6 +3525,7 @@ "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.8.4.tgz", "integrity": "sha512-LA4ZjwlnUblHVgq0oBF3Jl/6h/Nvs5fzBLwdEF4nuxnFdsfajde4WfxtJr3CaiH+F6ewcIB/q4jQ4UzPyid+CQ==", "license": "MIT", + "peer": true, "bin": { "node-gyp-build": "bin.js", "node-gyp-build-optional": "optional.js", @@ -3728,7 +3747,6 @@ "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, @@ -3785,7 +3803,6 @@ } ], "license": "MIT", - "peer": true, "dependencies": { "nanoid": "^3.3.11", "picocolors": "^1.1.1", @@ -3848,6 +3865,26 @@ "node": ">= 0.8.0" } }, + "node_modules/proper-lockfile": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/proper-lockfile/-/proper-lockfile-4.1.2.tgz", + "integrity": "sha512-TjNPblN4BwAWMXU8s9AEz4JmQxnD1NNL7bNOY/AKUzyamc379FWASUhc/K1pL2noVb+XmZKLL68cjzLsiOAMaA==", + "license": "MIT", + "dependencies": { + "graceful-fs": "^4.2.4", + "retry": "^0.12.0", + "signal-exit": "^3.0.2" + } + }, + "node_modules/proper-lockfile/node_modules/retry": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz", + "integrity": "sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==", + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, "node_modules/punycode": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", @@ -3996,6 +4033,12 @@ "dev": true, "license": "ISC" }, + "node_modules/signal-exit": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", + "license": "ISC" + }, "node_modules/simple-wcswidth": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/simple-wcswidth/-/simple-wcswidth-1.1.2.tgz", @@ -4311,7 +4354,6 @@ "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", "dev": true, "license": "Apache-2.0", - "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -4634,7 +4676,6 @@ "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", "license": "MIT", - "peer": true, "funding": { "url": "https://github.com/sponsors/colinhacks" } diff --git a/package.json b/package.json index 6b82c8c..cc16601 100644 --- a/package.json +++ b/package.json @@ -44,11 +44,13 @@ "url": "https://github.com/memvid/claude-brain/issues" }, "dependencies": { - "@memvid/sdk": "^2.0.146" + "@memvid/sdk": "^2.0.146", + "proper-lockfile": "^4.1.2" }, "devDependencies": { "@eslint/js": "^9.39.2", "@types/node": "^22.0.0", + "@types/proper-lockfile": "^4.1.4", "eslint": "^9.0.0", "tsup": "^8.0.0", "typescript": "^5.7.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ebe97b4..2689e6e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -11,6 +11,9 @@ importers: '@memvid/sdk': specifier: ^2.0.146 version: 2.0.146(@langchain/core@1.1.8(openai@6.15.0(zod@4.3.5)))(@llamaindex/env@0.1.30)(openai@6.15.0(zod@4.3.5))(zod@4.3.5) + proper-lockfile: + specifier: ^4.1.2 + version: 4.1.2 devDependencies: '@eslint/js': specifier: ^9.39.2 @@ -18,6 +21,9 @@ importers: '@types/node': specifier: ^22.0.0 version: 22.19.3 + '@types/proper-lockfile': + specifier: ^4.1.4 + version: 4.1.4 eslint: specifier: ^9.0.0 version: 9.39.2 @@ -560,6 +566,9 @@ packages: '@types/node@24.10.4': resolution: {integrity: sha512-vnDVpYPMzs4wunl27jHrfmwojOGKya0xyM3sH+UE5iv5uPS6vX7UIoh6m+vQc5LGBq52HBKPIn/zcSZVzeDEZg==} + '@types/proper-lockfile@4.1.4': + resolution: {integrity: sha512-uo2ABllncSqg9F1D4nugVl9v93RmjxF6LJzQLMLDdPaXCUIDPeOJ21Gbqi43xNKzBi/WQ0Q0dICqufzQbMjipQ==} + '@types/retry@0.12.0': resolution: {integrity: sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==} @@ -892,6 +901,9 @@ packages: resolution: {integrity: sha512-oahGvuMGQlPw/ivIYBjVSrWAfWLBeku5tpPE2fOPLi+WHffIWbuh2tCjhyQhTBPMf5E9jDEH4FOmTYgYwbKwtQ==} engines: {node: '>=18'} + graceful-fs@4.2.11: + resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==} + has-flag@4.0.0: resolution: {integrity: sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==} engines: {node: '>=8'} @@ -1152,6 +1164,9 @@ packages: resolution: {integrity: sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==} engines: {node: '>= 0.8.0'} + proper-lockfile@4.1.2: + resolution: {integrity: sha512-TjNPblN4BwAWMXU8s9AEz4JmQxnD1NNL7bNOY/AKUzyamc379FWASUhc/K1pL2noVb+XmZKLL68cjzLsiOAMaA==} + punycode@2.3.1: resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==} engines: {node: '>=6'} @@ -1168,6 +1183,10 @@ packages: resolution: {integrity: sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==} engines: {node: '>=8'} + retry@0.12.0: + resolution: {integrity: sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==} + engines: {node: '>= 4'} + retry@0.13.1: resolution: {integrity: sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==} engines: {node: '>= 4'} @@ -1196,6 +1215,9 @@ packages: siginfo@2.0.0: resolution: {integrity: sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==} + signal-exit@3.0.7: + resolution: {integrity: sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==} + simple-wcswidth@1.1.2: resolution: {integrity: sha512-j7piyCjAeTDSjzTSQ7DokZtMNwNlEAyxqSZeCS+CXH7fJ4jx3FuJ/mTW3mE+6JLs4VJBbcll0Kjn+KXI5t21Iw==} @@ -1846,6 +1868,10 @@ snapshots: dependencies: undici-types: 7.16.0 + '@types/proper-lockfile@4.1.4': + dependencies: + '@types/retry': 0.12.0 + '@types/retry@0.12.0': {} '@types/uuid@10.0.0': {} @@ -2233,6 +2259,8 @@ snapshots: globals@14.0.0: {} + graceful-fs@4.2.11: {} + has-flag@4.0.0: {} ignore@5.3.2: {} @@ -2443,6 +2471,12 @@ snapshots: prelude-ls@1.2.1: {} + proper-lockfile@4.1.2: + dependencies: + graceful-fs: 4.2.11 + retry: 0.12.0 + signal-exit: 3.0.7 + punycode@2.3.1: {} readdirp@4.1.2: {} @@ -2451,6 +2485,8 @@ snapshots: resolve-from@5.0.0: {} + retry@0.12.0: {} + retry@0.13.1: {} rollup@4.55.1: @@ -2496,6 +2532,8 @@ snapshots: siginfo@2.0.0: {} + signal-exit@3.0.7: {} + simple-wcswidth@1.1.2: {} source-map-js@1.2.1: {} diff --git a/src/__tests__/mind-lock.test.ts b/src/__tests__/mind-lock.test.ts new file mode 100644 index 0000000..e29a5bb --- /dev/null +++ b/src/__tests__/mind-lock.test.ts @@ -0,0 +1,60 @@ +import { describe, it, expect } from "vitest"; +import { Mind } from "../core/mind.js"; +import { mkdtempSync, rmSync, readdirSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; + +function makeTempMemoryPath(): { dir: string; path: string } { + const dir = mkdtempSync(join(tmpdir(), "claude-brain-lock-")); + return { dir, path: join(dir, "mind.mv2") }; +} + +async function writeOnce(memoryPath: string, i: number): Promise { + const mind = await Mind.open({ memoryPath, debug: false }); + await mind.remember({ + type: "discovery", + summary: `summary-${i}`, + content: `content-${i}`, + }); +} + +describe("Mind concurrent access", () => { + it("writes all frames in the happy path (single writer)", async () => { + const { dir, path } = makeTempMemoryPath(); + try { + const writes = 5; + for (let i = 0; i < writes; i++) { + await writeOnce(path, i); + } + + const mind = await Mind.open({ memoryPath: path, debug: false }); + const stats = await mind.stats(); + expect(stats.totalObservations).toBe(writes); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("preserves all frames with concurrent writers (edge case)", async () => { + const { dir, path } = makeTempMemoryPath(); + try { + const writes = 20; + const tasks = Array.from({ length: writes }, (_, i) => writeOnce(path, i)); + const results = await Promise.allSettled(tasks); + + const failed = results.filter((r) => r.status === "rejected"); + if (failed.length) { + throw failed[0].reason; + } + + const mind = await Mind.open({ memoryPath: path, debug: false }); + const stats = await mind.stats(); + expect(stats.totalObservations).toBe(writes); + + const backups = readdirSync(dir).filter((f) => f.includes(".backup-")); + expect(backups.length).toBe(0); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }, 15000); +}); diff --git a/src/core/mind.ts b/src/core/mind.ts index a25058c..d5cd881 100644 --- a/src/core/mind.ts +++ b/src/core/mind.ts @@ -23,6 +23,7 @@ import { DEFAULT_CONFIG, } from "../types.js"; import { generateId, estimateTokens } from "../utils/helpers.js"; +import { withMemvidLock } from "../utils/memvid-lock.js"; // Lazy-loaded SDK functions let sdkLoaded = false; @@ -84,12 +85,16 @@ export class Mind { await mkdir(memoryDir, { recursive: true }); // Open or create the memvid file - let memvid; + let memvid: Memvid; const MAX_FILE_SIZE_MB = 100; // Files over 100MB are likely corrupted + const lockPath = `${memoryPath}.lock`; + + await withMemvidLock(lockPath, async () => { + if (!existsSync(memoryPath)) { + memvid = await create(memoryPath, "basic"); + return; + } - if (!existsSync(memoryPath)) { - memvid = await create(memoryPath, "basic"); - } else { // Check file size - very large files are likely corrupted and will hang const { statSync, renameSync, unlinkSync } = await import("node:fs"); const fileSize = statSync(memoryPath).size; @@ -100,30 +105,31 @@ export class Mind { const backupPath = `${memoryPath}.backup-${Date.now()}`; try { renameSync(memoryPath, backupPath); } catch { /* ignore */ } memvid = await create(memoryPath, "basic"); - } else { - try { - memvid = await use("basic", memoryPath); - } catch (openError: unknown) { - const errorMessage = openError instanceof Error ? openError.message : String(openError); - // Handle corrupted or incompatible memory files - if (errorMessage.includes("Deserialization") || - errorMessage.includes("UnexpectedVariant") || - errorMessage.includes("Invalid") || - errorMessage.includes("corrupt")) { - console.error("[memvid-mind] Memory file corrupted, creating fresh memory..."); - const backupPath = `${memoryPath}.backup-${Date.now()}`; - try { - renameSync(memoryPath, backupPath); - } catch { - try { unlinkSync(memoryPath); } catch { /* ignore */ } - } - memvid = await create(memoryPath, "basic"); - } else { - throw openError; + return; + } + + try { + memvid = await use("basic", memoryPath); + } catch (openError: unknown) { + const errorMessage = openError instanceof Error ? openError.message : String(openError); + // Handle corrupted or incompatible memory files + if (errorMessage.includes("Deserialization") || + errorMessage.includes("UnexpectedVariant") || + errorMessage.includes("Invalid") || + errorMessage.includes("corrupt")) { + console.error("[memvid-mind] Memory file corrupted, creating fresh memory..."); + const backupPath = `${memoryPath}.backup-${Date.now()}`; + try { + renameSync(memoryPath, backupPath); + } catch { + try { unlinkSync(memoryPath); } catch { /* ignore */ } } + memvid = await create(memoryPath, "basic"); + return; } + throw openError; } - } + }); const mind = new Mind(memvid, config); mind.initialized = true; @@ -135,6 +141,12 @@ export class Mind { return mind; } + private async withLock(fn: () => Promise): Promise { + const memoryPath = this.getMemoryPath(); + const lockPath = `${memoryPath}.lock`; + return withMemvidLock(lockPath, fn); + } + /** * Remember an observation */ @@ -158,19 +170,20 @@ export class Mind { }, }; - // Store in memvid - const frameId = await this.memvid.put({ - title: `[${observation.type}] ${observation.summary}`, - label: observation.type, - text: observation.content, - metadata: { - observationId: observation.id, - timestamp: observation.timestamp, - tool: observation.tool, - sessionId: this.sessionId, - ...observation.metadata, - }, - tags: [observation.type, observation.tool].filter(Boolean) as string[], + const frameId = await this.withLock(async () => { + return this.memvid.put({ + title: `[${observation.type}] ${observation.summary}`, + label: observation.type, + text: observation.content, + metadata: { + observationId: observation.id, + timestamp: observation.timestamp, + tool: observation.tool, + sessionId: this.sessionId, + ...observation.metadata, + }, + tags: [observation.type, observation.tool].filter(Boolean) as string[], + }); }); if (this.config.debug) { @@ -184,6 +197,12 @@ export class Mind { * Search memories by query (uses fast lexical search) */ async search(query: string, limit = 10): Promise { + return this.withLock(async () => { + return this.searchUnlocked(query, limit); + }); + } + + private async searchUnlocked(query: string, limit: number): Promise { const results = await this.memvid.find(query, { k: limit, mode: "lex" }); return (results.frames || []).map((frame: any) => ({ @@ -205,69 +224,73 @@ export class Mind { * Ask the memory a question (uses fast lexical search) */ async ask(question: string): Promise { - const result = await this.memvid.ask(question, { k: 5, mode: "lex" }); - return result.answer || "No relevant memories found."; + return this.withLock(async () => { + const result = await this.memvid.ask(question, { k: 5, mode: "lex" }); + return result.answer || "No relevant memories found."; + }); } /** * Get context for session start */ async getContext(query?: string): Promise { - // Get recent observations via timeline - const timeline = await this.memvid.timeline({ - limit: this.config.maxContextObservations, - reverse: true, - }); - - // SDK returns array directly or { frames: [...] } - const frames = Array.isArray(timeline) ? timeline : (timeline.frames || []); - - const recentObservations: Observation[] = frames.map( - (frame: any) => { - // Get timestamp - SDK returns seconds, convert to milliseconds if needed - let ts = frame.metadata?.timestamp || frame.timestamp || 0; - // If timestamp looks like seconds (before year 2100 in seconds), convert to ms - if (ts > 0 && ts < 4102444800) { - ts = ts * 1000; + return this.withLock(async () => { + // Get recent observations via timeline + const timeline = await this.memvid.timeline({ + limit: this.config.maxContextObservations, + reverse: true, + }); + + // SDK returns array directly or { frames: [...] } + const frames = Array.isArray(timeline) ? timeline : (timeline.frames || []); + + const recentObservations: Observation[] = frames.map( + (frame: any) => { + // Get timestamp - SDK returns seconds, convert to milliseconds if needed + let ts = frame.metadata?.timestamp || frame.timestamp || 0; + // If timestamp looks like seconds (before year 2100 in seconds), convert to ms + if (ts > 0 && ts < 4102444800) { + ts = ts * 1000; + } + return { + id: frame.metadata?.observationId || frame.frame_id, + timestamp: ts, + type: (frame.label || frame.metadata?.type || "observation") as ObservationType, + tool: frame.metadata?.tool, + summary: frame.title?.replace(/^\[.*?\]\s*/, "") || frame.preview?.slice(0, 100) || "", + content: frame.text || frame.preview || "", + metadata: frame.metadata, + }; } - return { - id: frame.metadata?.observationId || frame.frame_id, - timestamp: ts, - type: (frame.label || frame.metadata?.type || "observation") as ObservationType, - tool: frame.metadata?.tool, - summary: frame.title?.replace(/^\[.*?\]\s*/, "") || frame.preview?.slice(0, 100) || "", - content: frame.text || frame.preview || "", - metadata: frame.metadata, - }; - } - ); + ); - // Get relevant memories if query provided - let relevantMemories: Observation[] = []; - if (query) { - const searchResults = await this.search(query, 10); - relevantMemories = searchResults.map((r) => r.observation); - } + // Get relevant memories if query provided + let relevantMemories: Observation[] = []; + if (query) { + const searchResults = await this.searchUnlocked(query, 10); + relevantMemories = searchResults.map((r) => r.observation); + } - // Build context with token limit - const contextParts: string[] = []; - let tokenCount = 0; - - // Add recent observations - for (const obs of recentObservations) { - const text = `[${obs.type}] ${obs.summary}`; - const tokens = estimateTokens(text); - if (tokenCount + tokens > this.config.maxContextTokens) break; - contextParts.push(text); - tokenCount += tokens; - } + // Build context with token limit + const contextParts: string[] = []; + let tokenCount = 0; + + // Add recent observations + for (const obs of recentObservations) { + const text = `[${obs.type}] ${obs.summary}`; + const tokens = estimateTokens(text); + if (tokenCount + tokens > this.config.maxContextTokens) break; + contextParts.push(text); + tokenCount += tokens; + } - return { - recentObservations, - relevantMemories, - sessionSummaries: [], // TODO: Implement session summaries - tokenCount, - }; + return { + recentObservations, + relevantMemories, + sessionSummaries: [], // TODO: Implement session summaries + tokenCount, + }; + }); } /** @@ -288,12 +311,14 @@ export class Mind { summary: summary.summary, }; - return this.memvid.put({ - title: `Session Summary: ${new Date().toISOString().split("T")[0]}`, - label: "session", - text: JSON.stringify(sessionSummary, null, 2), - metadata: sessionSummary as unknown as Record, - tags: ["session", "summary"], + return this.withLock(async () => { + return this.memvid.put({ + title: `Session Summary: ${new Date().toISOString().split("T")[0]}`, + label: "session", + text: JSON.stringify(sessionSummary, null, 2), + metadata: sessionSummary as unknown as Record, + tags: ["session", "summary"], + }); }); } @@ -301,22 +326,24 @@ export class Mind { * Get memory statistics */ async stats(): Promise { - const stats = await this.memvid.stats(); - const timeline = await this.memvid.timeline({ limit: 1, reverse: false }); - const recentTimeline = await this.memvid.timeline({ limit: 1, reverse: true }); - - // SDK returns array directly or { frames: [...] } - const oldestFrames = Array.isArray(timeline) ? timeline : (timeline.frames || []); - const newestFrames = Array.isArray(recentTimeline) ? recentTimeline : (recentTimeline.frames || []); - - return { - totalObservations: (stats.frame_count as number) || 0, - totalSessions: 0, // TODO: Count unique sessions - oldestMemory: (oldestFrames[0] as any)?.metadata?.timestamp || (oldestFrames[0] as any)?.timestamp || 0, - newestMemory: (newestFrames[0] as any)?.metadata?.timestamp || (newestFrames[0] as any)?.timestamp || 0, - fileSize: (stats.size_bytes as number) || 0, - topTypes: {} as Record, // TODO: Aggregate - }; + return this.withLock(async () => { + const stats = await this.memvid.stats(); + const timeline = await this.memvid.timeline({ limit: 1, reverse: false }); + const recentTimeline = await this.memvid.timeline({ limit: 1, reverse: true }); + + // SDK returns array directly or { frames: [...] } + const oldestFrames = Array.isArray(timeline) ? timeline : (timeline.frames || []); + const newestFrames = Array.isArray(recentTimeline) ? recentTimeline : (recentTimeline.frames || []); + + return { + totalObservations: (stats.frame_count as number) || 0, + totalSessions: 0, // TODO: Count unique sessions + oldestMemory: (oldestFrames[0] as any)?.metadata?.timestamp || (oldestFrames[0] as any)?.timestamp || 0, + newestMemory: (newestFrames[0] as any)?.metadata?.timestamp || (newestFrames[0] as any)?.timestamp || 0, + fileSize: (stats.size_bytes as number) || 0, + topTypes: {} as Record, // TODO: Aggregate + }; + }); } /** diff --git a/src/utils/memvid-lock.ts b/src/utils/memvid-lock.ts new file mode 100644 index 0000000..05f6fae --- /dev/null +++ b/src/utils/memvid-lock.ts @@ -0,0 +1,28 @@ +import lockfile from "proper-lockfile"; +import { mkdir, open } from "node:fs/promises"; +import { dirname } from "node:path"; + +const LOCK_OPTIONS = { + stale: 30000, + retries: { + retries: 1000, + minTimeout: 5, + maxTimeout: 50, + }, +} as const; + +export async function withMemvidLock( + lockPath: string, + fn: () => Promise +): Promise { + await mkdir(dirname(lockPath), { recursive: true }); + const handle = await open(lockPath, "a"); + await handle.close(); + + const release = await lockfile.lock(lockPath, LOCK_OPTIONS); + try { + return await fn(); + } finally { + await release(); + } +}