feat: native SQLite and Postgres database support based on Drizzle ORM (openwallet-foundation#2324)

Signed-off-by: Timo Glastra <timo@animo.id>

Author: TimoGlastra

.changeset/kind-bobcats-camp.md

@@ -0,0 +1,9 @@
+---
+"@credo-ts/drizzle-storage": minor
+---
+
+feat: add support for new SQLite and PostgreSQL storage based on Drizzle.
+
+The Drizzle Storage Module is an additional storage implementation for Credo which natively integrates with PostgreSQL and SQLite. It can be combined with Aries Askar as the KMS.
+
+The Drizzle Storage Module does not introduce any breaking chnages to how the storage APIs works in Credo, and for new projects you only have to configure the Drizzle module to connect to your database.

.github/workflows/continuous-integration.yml

@@ -59,7 +59,7 @@ jobs:
         # Each shard runs a set of the tests
         # Make sure to UPDATE THE TEST command with the total length of
         # the shards if you change this!!
-        shard: [1, 2]
+        shard: [1, 2, 3]
 
     steps:
       - uses: actions/checkout@v5
@@ -88,14 +88,21 @@ jobs:
       - name: Install dependencies
         run: pnpm install --frozen-lockfile
 
+      # Issue with node 22 starting 22.18
+      # https://github.com/nodejs/node/issues/59364
       - name: Run tests
-        run: pnpm test:unit --coverage --forceExit --shard=${{ matrix.shard }}/2
+        run: NODE_OPTIONS="$NODE_OPTIONS --no-experimental-strip-types" pnpm test:unit --coverage --forceExit --shard=${{ matrix.shard }}/3
+        if: matrix.node-version == 22
+
+      - name: Run tests
+        run: pnpm test:unit --coverage --forceExit --shard=${{ matrix.shard }}/3
+        if: matrix.node-version != 22
 
       # Upload coverage for shard
       - run: mv coverage/coverage-final.json coverage/${{ matrix.shard }}.json
       - uses: actions/upload-artifact@v4
         with:
-          name: coverage-artifacts-${{ matrix.node-version }}
+          name: coverage-artifacts-${{ matrix.node-version }}-unit-${{ matrix.shard }}
           path: coverage/${{ matrix.shard }}.json
           overwrite: true
 
@@ -141,26 +148,94 @@ jobs:
       - name: Install dependencies
         run: pnpm install --frozen-lockfile
 
+      # Issue with node 22 starting 22.18
+      # https://github.com/nodejs/node/issues/59364
+      - name: Run tests
+        run: NODE_OPTIONS="$NODE_OPTIONS --no-experimental-strip-types" pnpm test:e2e --coverage --forceExit
+        if: matrix.node-version == 22
+
       - name: Run tests
         run: pnpm test:e2e --coverage --forceExit
+        if: matrix.node-version != 22
 
       # Upload coverage for e2e
       - run: mv coverage/coverage-final.json coverage/e2e.json
       - uses: actions/upload-artifact@v4
         with:
-          name: coverage-artifacts-${{ matrix.node-version }}
+          name: coverage-artifacts-${{ matrix.node-version }}-e2e
           path: coverage/e2e.json
           overwrite: true
 
+  drizzle-tests:
+    runs-on: ubuntu-24.04
+    name: Drizzle Tests
+    if: github.event_name != 'pull_request_review' ||  github.event.pull_request.head.ref == 'changeset-release/main'
+
+    strategy:
+      fail-fast: false
+      matrix:
+        node-version: [20, 22]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      # setup dependencies
+      - name: Setup services
+        run: docker compose up -d
+
+      - name: Setup NodeJS
+        id: setup-node
+        uses: actions/setup-node@v4
+        with:
+          node-version: ${{ matrix.node-version }}
+
+      - uses: pnpm/action-setup@v4
+
+      # See https://github.com/actions/setup-node/issues/641#issuecomment-1358859686
+      - name: pnpm cache path
+        id: pnpm-cache-path
+        run: |
+          echo "STORE_PATH=$(pnpm store path)" >> $GITHUB_OUTPUT
+
+      - name: pnpm cache
+        uses: actions/cache@v3
+        with:
+          path: ${{ steps.pnpm-cache-path.outputs.STORE_PATH }}
+          key: ${{ runner.os }}-${{ steps.setup-node.outputs.node-version }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
+          restore-keys: |
+            ${{ runner.os }}-${{ steps.setup-node.outputs.node-version }}-pnpm-store-
+
+      - name: Install dependencies
+        run: pnpm install --frozen-lockfile
+
+      # Issue with node 22 starting 22.18
+      # https://github.com/nodejs/node/issues/59364
+      - name: Run tests
+        run: NODE_OPTIONS="$NODE_OPTIONS --no-experimental-strip-types" pnpm test:drizzle --coverage --forceExit
+        if: matrix.node-version == 22
+
+      - name: Run tests
+        run: pnpm test:drizzle --coverage --forceExit
+        if: matrix.node-version != 22
+
+      # Upload coverage for e2e
+      - run: mv coverage/coverage-final.json coverage/drizzle.json
+      - uses: actions/upload-artifact@v4
+        with:
+          name: coverage-artifacts-${{ matrix.node-version }}-drizzle
+          path: coverage/drizzle.json
+          overwrite: true
+
   # Upload all the coverage reports
   report-coverage:
     runs-on: ubuntu-24.04
-    needs: [e2e-tests, unit-tests]
+    needs: [e2e-tests, unit-tests, drizzle-tests]
     steps:
       - uses: actions/download-artifact@v5
         with:
-          name: coverage-artifacts-20
+          pattern: coverage-artifacts-20-*
           path: coverage
+          merge-multiple: true
 
       - uses: codecov/codecov-action@v5
         with:

.gitignore

@@ -11,4 +11,6 @@ coverage
 logs.txt
 logs/
 .pnpm-store
-ngrok.auth.yml
+ngrok.auth.yml
+cli-build
+*.db

jest.config.base.ts

@@ -6,14 +6,13 @@ const config: Config.InitialOptions = {
   // NOTE: overridden in e2e test. Make sure to
   // update that match as well when changing this one
   testMatch: ['**/?(*.)test.ts'],
-  moduleNameMapper: {
-    '@credo-ts/(.+)': ['<rootDir>/../../packages/$1/src', '<rootDir>/../packages/$1/src', '<rootDir>/packages/$1/src'],
-  },
   transform: {
-    '\\.tsx?$': [
+    '\\.(t|j)sx?$': [
       'ts-jest',
       {
-        isolatedModules: true,
+        tsconfig: {
+          isolatedModules: true,
+        },
       },
     ],
   },

jest.config.ts

@@ -5,6 +5,7 @@ import base from './jest.config.base'
 const config: Config.InitialOptions = {
   ...base,
   roots: ['<rootDir>'],
+  verbose: true,
   coverageReporters: ['text-summary', 'lcov', 'json'],
   coveragePathIgnorePatterns: ['/build/', '/node_modules/', '/__tests__/', 'tests'],
   coverageDirectory: '<rootDir>/coverage/',

package.json

@@ -15,8 +15,9 @@
     "style:fix": "biome check --write --unsafe",
     "clean": "pnpm -r --parallel run clean",
     "build": "pnpm -r --parallel run build",
-    "test:unit": "jest --testPathIgnorePatterns 'e2e.test.ts$'",
-    "test:e2e": "jest --testMatch '**/?(*.)e2e.test.ts'",
+    "test:unit": "pnpm test --testPathIgnorePatterns e2e.test.ts",
+    "test:e2e": "pnpm test --testPathIgnorePatterns drizzle --testMatch '**/?(*.)e2e.test.ts'",
+    "test:drizzle": "pnpm test --testMatch '**/*drizzle*.test.ts'",
     "test": "jest",
     "validate": "pnpm types:check && pnpm style:check",
     "run-mediator": "ts-node ./samples/mediator.ts",
@@ -28,22 +29,38 @@
     "@babel/preset-env": "^7.27.2",
     "@biomejs/biome": "^1.9.4",
     "@changesets/cli": "^2.29.4",
+    "@credo-ts/action-menu": "workspace:*",
+    "@credo-ts/anoncreds": "workspace:*",
+    "@credo-ts/askar": "workspace:*",
+    "@credo-ts/cheqd": "workspace:*",
+    "@credo-ts/core": "workspace:*",
+    "@credo-ts/didcomm": "workspace:*",
+    "@credo-ts/drizzle-storage": "workspace:*",
+    "@credo-ts/drpc": "workspace:*",
+    "@credo-ts/indy-sdk-to-askar-migration": "workspace:*",
+    "@credo-ts/indy-vdr": "workspace:*",
+    "@credo-ts/node": "workspace:*",
+    "@credo-ts/openid4vc": "workspace:*",
+    "@credo-ts/question-answer": "workspace:*",
+    "@credo-ts/react-native": "workspace:*",
+    "@credo-ts/redis-cache": "workspace:*",
+    "@credo-ts/tenants": "workspace:*",
     "@jest/types": "^29.6.3",
     "@openwallet-foundation/askar-nodejs": "catalog:",
     "@types/cors": "^2.8.19",
     "@types/express": "catalog:",
-    "@types/jest": "^29.5.14",
+    "@types/jest": "^30.0.0",
     "@types/node": "catalog:",
     "@types/supertest": "^6.0.3",
     "@types/varint": "^6.0.3",
     "@types/ws": "catalog:",
     "cors": "^2.8.5",
     "express": "catalog:",
-    "jest": "^29.7.0",
+    "jest": "^30.0.3",
     "nock": "catalog:",
     "rxjs": "catalog:",
-    "supertest": "^7.1.1",
-    "ts-jest": "^29.3.4",
+    "supertest": "^7.1.3",
+    "ts-jest": "^29.4.0",
     "ts-node": "catalog:",
     "tsyringe": "catalog:",
     "typescript": "catalog:",

packages/action-menu/src/services/__tests__/ActionMenuService.test.ts

@@ -94,7 +94,7 @@ describe('ActionMenuService', () => {
             ],
           },
         })
-      ).rejects.toThrowError('Action Menu contains duplicated options')
+      ).rejects.toThrow('Action Menu contains duplicated options')
     })
 
     it('no previous menu: emits a menu with title, description and options', async () => {
@@ -241,7 +241,7 @@ describe('ActionMenuService', () => {
           actionMenuRecord: mockRecord,
           performedAction: { name: 'fake' },
         })
-      ).rejects.toThrowError('Selection does not match valid actions')
+      ).rejects.toThrow('Selection does not match valid actions')
     })
 
     it('throws an error when state is not preparing-selection', async () => {
@@ -254,9 +254,7 @@ describe('ActionMenuService', () => {
             actionMenuRecord: mockRecord,
             performedAction: { name: 'opt1' },
           })
-        ).rejects.toThrowError(
-          `Action Menu record is in invalid state ${state}. Valid states are: preparing-selection.`
-        )
+        ).rejects.toThrow(`Action Menu record is in invalid state ${state}. Valid states are: preparing-selection.`)
       }
     })
 
@@ -696,9 +694,7 @@ describe('ActionMenuService', () => {
 
       mockFunction(actionMenuRepository.findSingleByQuery).mockReturnValue(Promise.resolve(mockRecord))
 
-      expect(actionMenuService.processPerform(messageContext)).rejects.toThrowError(
-        'Selection does not match valid actions'
-      )
+      expect(actionMenuService.processPerform(messageContext)).rejects.toThrow('Selection does not match valid actions')
 
       expect(actionMenuRepository.update).not.toHaveBeenCalled()
       expect(actionMenuRepository.save).not.toHaveBeenCalled()
@@ -721,7 +717,7 @@ describe('ActionMenuService', () => {
 
       mockFunction(actionMenuRepository.findSingleByQuery).mockReturnValue(Promise.resolve(null))
 
-      expect(actionMenuService.processPerform(messageContext)).rejects.toThrowError(
+      expect(actionMenuService.processPerform(messageContext)).rejects.toThrow(
         `No Action Menu found with thread id ${mockPerformMessage.threadId}`
       )
 
@@ -747,7 +743,7 @@ describe('ActionMenuService', () => {
       mockRecord.state = ActionMenuState.Done
       mockFunction(actionMenuRepository.findSingleByQuery).mockReturnValue(Promise.resolve(mockRecord))
 
-      expect(actionMenuService.processPerform(messageContext)).rejects.toThrowError(
+      expect(actionMenuService.processPerform(messageContext)).rejects.toThrow(
         `Action Menu record is in invalid state ${mockRecord.state}. Valid states are: ${ActionMenuState.AwaitingSelection}.`
       )
 
@@ -773,7 +769,7 @@ describe('ActionMenuService', () => {
       mockRecord.state = ActionMenuState.Null
       mockFunction(actionMenuRepository.findSingleByQuery).mockReturnValue(Promise.resolve(mockRecord))
 
-      expect(actionMenuService.processPerform(messageContext)).rejects.toThrowError(
+      expect(actionMenuService.processPerform(messageContext)).rejects.toThrow(
         new ActionMenuProblemReportError('Action Menu has been cleared by the responder', {
           problemCode: ActionMenuProblemReportReason.Timeout,
         })

packages/anoncreds/src/anoncreds-rs/__tests__/AnonCredsRsHolderService.test.ts

@@ -361,7 +361,7 @@ describe('AnonCredsRsHolderService', () => {
           proofRequest,
           attributeReferent: 'name',
         })
-      ).rejects.toThrowError()
+      ).rejects.toThrow()
     })
 
     test('referent with single restriction', async () => {
@@ -513,7 +513,7 @@ describe('AnonCredsRsHolderService', () => {
     findByIdMock.mockResolvedValueOnce(null).mockResolvedValueOnce(record)
     getByCredentialIdMock.mockRejectedValueOnce(new Error())
 
-    await expect(anonCredsHolderService.getCredential(agentContext, { id: 'myCredentialId' })).rejects.toThrowError()
+    await expect(anonCredsHolderService.getCredential(agentContext, { id: 'myCredentialId' })).rejects.toThrow()
 
     const credentialInfo = await anonCredsHolderService.getCredential(agentContext, { id: 'myCredentialId' })
 

packages/anoncreds/src/protocols/credentials/v1/__tests__/V1CredentialProtocolProposeOffer.test.ts

@@ -296,7 +296,7 @@ describe('V1CredentialProtocolProposeOffer', () => {
         }),
       })
 
-      await expect(credentialProtocol.createOffer(agentContext, offerOptions)).rejects.toThrowError(
+      await expect(credentialProtocol.createOffer(agentContext, offerOptions)).rejects.toThrow(
         'Missing required credential preview from indy format service'
       )
     })

packages/anoncreds/src/protocols/proofs/v1/__tests__/v1-indy-proofs.e2e.test.ts

@@ -482,7 +482,7 @@ describe('Present Proof', () => {
           },
         },
       })
-    ).rejects.toThrowError('The proof request contains duplicate predicates and attributes: age')
+    ).rejects.toThrow('The proof request contains duplicate predicates and attributes: age')
   })
 
   test('Faber starts with proof request to Alice but gets Problem Reported', async () => {