diff --git a/docs/api/classes/modules_assets_assets_service.AssetsService.md b/docs/api/classes/modules_assets_assets_service.AssetsService.md index 865bf113c..c2869767d 100644 --- a/docs/api/classes/modules_assets_assets_service.AssetsService.md +++ b/docs/api/classes/modules_assets_assets_service.AssetsService.md @@ -65,7 +65,7 @@ ___ | :------ | :------ | | `assetId` | `string` | | `«destructured»` | `Object` | -| › `order` | ``"ASC"`` \| ``"DESC"`` | +| › `order` | ``"DESC"`` \| ``"ASC"`` | | › `skip` | `number` | | › `take` | `number` | | › `type` | [`AssetHistoryEventType`](../enums/modules_assets_assets_event.AssetHistoryEventType.md) | diff --git a/docs/api/classes/modules_auth_login_strategy.AuthStrategy.md b/docs/api/classes/modules_auth_login_strategy.AuthStrategy.md index 5844f88e5..570a77fab 100644 --- a/docs/api/classes/modules_auth_login_strategy.AuthStrategy.md +++ b/docs/api/classes/modules_auth_login_strategy.AuthStrategy.md @@ -14,19 +14,6 @@ - [constructor](modules_auth_login_strategy.AuthStrategy.md#constructor) -### Methods - -- [authenticate](modules_auth_login_strategy.AuthStrategy.md#authenticate) -- [decodeToken](modules_auth_login_strategy.AuthStrategy.md#decodetoken) -- [didUnification](modules_auth_login_strategy.AuthStrategy.md#didunification) -- [extractSiwe](modules_auth_login_strategy.AuthStrategy.md#extractsiwe) -- [extractToken](modules_auth_login_strategy.AuthStrategy.md#extracttoken) -- [getRoleDefinition](modules_auth_login_strategy.AuthStrategy.md#getroledefinition) -- [isEIP191TokenPayload](modules_auth_login_strategy.AuthStrategy.md#iseip191tokenpayload) -- [isSiweMessagePayload](modules_auth_login_strategy.AuthStrategy.md#issiwemessagepayload) -- [validate](modules_auth_login_strategy.AuthStrategy.md#validate) -- [verifyIssuer](modules_auth_login_strategy.AuthStrategy.md#verifyissuer) - ## Constructors ### constructor @@ -46,213 +33,3 @@ #### Overrides PassportStrategy(LoginStrategy, 'login').constructor - -## Methods - -### authenticate - -▸ **authenticate**(`req`): `void` - -#### Parameters - -| Name | Type | -| :------ | :------ | -| `req` | `Request`<`ParamsDictionary`, `any`, `any`, `ParsedQs`, `Record`<`string`, `any`\>\> | - -#### Returns - -`void` - -#### Inherited from - -PassportStrategy(LoginStrategy, 'login').authenticate - -___ - -### decodeToken - -▸ **decodeToken**<`T`\>(`token`, `options?`): `T` - -#### Type parameters - -| Name | -| :------ | -| `T` | - -#### Parameters - -| Name | Type | -| :------ | :------ | -| `token` | `string` | -| `options?` | `DecodeOptions` | - -#### Returns - -`T` - -#### Inherited from - -PassportStrategy(LoginStrategy, 'login').decodeToken - -___ - -### didUnification - -▸ **didUnification**(`did`): `string` - -#### Parameters - -| Name | Type | -| :------ | :------ | -| `did` | `string` | - -#### Returns - -`string` - -#### Inherited from - -PassportStrategy(LoginStrategy, 'login').didUnification - -___ - -### extractSiwe - -▸ **extractSiwe**(`req`): `SiweReqPayload` - -#### Parameters - -| Name | Type | -| :------ | :------ | -| `req` | `Request`<`ParamsDictionary`, `any`, `any`, `ParsedQs`, `Record`<`string`, `any`\>\> | - -#### Returns - -`SiweReqPayload` - -#### Inherited from - -PassportStrategy(LoginStrategy, 'login').extractSiwe - -___ - -### extractToken - -▸ **extractToken**(`req`): `string` - -#### Parameters - -| Name | Type | -| :------ | :------ | -| `req` | `Request`<`ParamsDictionary`, `any`, `any`, `ParsedQs`, `Record`<`string`, `any`\>\> | - -#### Returns - -`string` - -#### Inherited from - -PassportStrategy(LoginStrategy, 'login').extractToken - -___ - -### getRoleDefinition - -▸ **getRoleDefinition**(`namespace`): `Promise`<`IRoleDefinitionV2`\> - -#### Parameters - -| Name | Type | -| :------ | :------ | -| `namespace` | `string` | - -#### Returns - -`Promise`<`IRoleDefinitionV2`\> - -#### Inherited from - -PassportStrategy(LoginStrategy, 'login').getRoleDefinition - -___ - -### isEIP191TokenPayload - -▸ **isEIP191TokenPayload**(`payload`): payload is ITokenPayload - -#### Parameters - -| Name | Type | -| :------ | :------ | -| `payload` | `unknown` | - -#### Returns - -payload is ITokenPayload - -#### Inherited from - -PassportStrategy(LoginStrategy, 'login').isEIP191TokenPayload - -___ - -### isSiweMessagePayload - -▸ **isSiweMessagePayload**(`payload`): payload is Partial - -#### Parameters - -| Name | Type | -| :------ | :------ | -| `payload` | `unknown` | - -#### Returns - -payload is Partial - -#### Inherited from - -PassportStrategy(LoginStrategy, 'login').isSiweMessagePayload - -___ - -### validate - -▸ **validate**(`token`, `payload`, `done`): `Promise`<`void`\> - -#### Parameters - -| Name | Type | -| :------ | :------ | -| `token` | `string` | -| `payload` | `Partial`<`SiweMessage`\> \| `ITokenPayload` | -| `done` | (`err?`: `Error`, `user?`: `unknown`, `info?`: `unknown`) => `void` | - -#### Returns - -`Promise`<`void`\> - -#### Inherited from - -PassportStrategy(LoginStrategy, 'login').validate - -___ - -### verifyIssuer - -▸ **verifyIssuer**(`issuer`, `role`): `Promise`<`VerificationResult`\> - -#### Parameters - -| Name | Type | -| :------ | :------ | -| `issuer` | `string` | -| `role` | `string` | - -#### Returns - -`Promise`<`VerificationResult`\> - -#### Inherited from - -PassportStrategy(LoginStrategy, 'login').verifyIssuer diff --git a/docs/api/classes/modules_did_didSyncStatus_entity.DidSyncStatusEntity.md b/docs/api/classes/modules_did_didSyncStatus_entity.DidSyncStatusEntity.md new file mode 100644 index 000000000..801f5093a --- /dev/null +++ b/docs/api/classes/modules_did_didSyncStatus_entity.DidSyncStatusEntity.md @@ -0,0 +1,46 @@ +# Class: DidSyncStatusEntity + +[modules/did/didSyncStatus.entity](../modules/modules_did_didSyncStatus_entity.md).DidSyncStatusEntity + +## Table of contents + +### Constructors + +- [constructor](modules_did_didSyncStatus_entity.DidSyncStatusEntity.md#constructor) + +### Properties + +- [createdDate](modules_did_didSyncStatus_entity.DidSyncStatusEntity.md#createddate) +- [document](modules_did_didSyncStatus_entity.DidSyncStatusEntity.md#document) +- [id](modules_did_didSyncStatus_entity.DidSyncStatusEntity.md#id) +- [status](modules_did_didSyncStatus_entity.DidSyncStatusEntity.md#status) + +## Constructors + +### constructor + +• **new DidSyncStatusEntity**() + +## Properties + +### createdDate + +• **createdDate**: `Date` + +___ + +### document + +• **document**: [`DIDDocumentEntity`](modules_did_did_entity.DIDDocumentEntity.md) + +___ + +### id + +• **id**: `number` + +___ + +### status + +• **status**: [`DidSyncStatus`](../enums/modules_did_did_types.DidSyncStatus.md) diff --git a/docs/api/classes/modules_did_did_processor.DIDProcessor.md b/docs/api/classes/modules_did_did_processor.DIDProcessor.md index aa8885f46..f0ddf5b38 100644 --- a/docs/api/classes/modules_did_did_processor.DIDProcessor.md +++ b/docs/api/classes/modules_did_did_processor.DIDProcessor.md @@ -22,7 +22,7 @@ ### constructor -• **new DIDProcessor**(`didService`, `logger`, `configService`, `pinQueue`) +• **new DIDProcessor**(`didService`, `logger`, `configService`) #### Parameters @@ -31,19 +31,18 @@ | `didService` | [`DIDService`](modules_did_did_service.DIDService.md) | | `logger` | [`Logger`](modules_logger_logger_service.Logger.md) | | `configService` | `ConfigService`<`Record`<`string`, `unknown`\>, ``false``\> | -| `pinQueue` | `Queue`<[`PinClaimData`](../modules/modules_ipfs_ipfs_types.md#pinclaimdata)\> | ## Methods ### OnQueueWaiting -▸ **OnQueueWaiting**(`job`): `Promise`<`void`\> +▸ **OnQueueWaiting**(`jobId`): `Promise`<`void`\> #### Parameters | Name | Type | | :------ | :------ | -| `job` | `Job`<`any`\> | +| `jobId` | `number` | #### Returns @@ -59,7 +58,7 @@ ___ | Name | Type | | :------ | :------ | -| `job` | `Job`<`any`\> | +| `job` | `Job`<[`UpdateDocumentJobData`](../modules/modules_did_did_types.md#updatedocumentjobdata)\> | #### Returns @@ -91,7 +90,7 @@ ___ | Name | Type | | :------ | :------ | -| `job` | `Job`<`any`\> | +| `job` | `Job`<[`UpdateDocumentJobData`](../modules/modules_did_did_types.md#updatedocumentjobdata)\> | #### Returns @@ -107,7 +106,7 @@ ___ | Name | Type | | :------ | :------ | -| `job` | `Job`<`any`\> | +| `job` | `Job`<[`UpdateDocumentJobData`](../modules/modules_did_did_types.md#updatedocumentjobdata)\> | #### Returns @@ -139,7 +138,7 @@ ___ | Name | Type | | :------ | :------ | -| `job` | `Job`<`string`\> | +| `job` | `Job`<[`UpdateDocumentJobData`](../modules/modules_did_did_types.md#updatedocumentjobdata)\> | #### Returns diff --git a/docs/api/classes/modules_did_did_service.DIDService.md b/docs/api/classes/modules_did_did_service.DIDService.md index c89568193..0fadd3187 100644 --- a/docs/api/classes/modules_did_did_service.DIDService.md +++ b/docs/api/classes/modules_did_did_service.DIDService.md @@ -28,7 +28,7 @@ ### constructor -• **new DIDService**(`config`, `schedulerRegistry`, `httpService`, `didQueue`, `logger`, `didRepository`, `provider`, `sentryTracingService`, `registrySettings`, `ipfsService`) +• **new DIDService**(`config`, `schedulerRegistry`, `httpService`, `didQueue`, `logger`, `didRepository`, `provider`, `sentryTracingService`, `registrySettings`, `ipfsService`, `latestDidSyncRepository`, `didSyncStatusRepository`, `dataSource`) #### Parameters @@ -37,13 +37,16 @@ | `config` | `ConfigService`<`Record`<`string`, `unknown`\>, ``false``\> | | `schedulerRegistry` | `SchedulerRegistry` | | `httpService` | `HttpService` | -| `didQueue` | `Queue`<`string`\> | +| `didQueue` | `Queue`<[`UpdateDocumentJobData`](../modules/modules_did_did_types.md#updatedocumentjobdata)\> | | `logger` | [`Logger`](modules_logger_logger_service.Logger.md) | | `didRepository` | `Repository`<[`DIDDocumentEntity`](modules_did_did_entity.DIDDocumentEntity.md)\> | | `provider` | [`Provider`](common_provider.Provider.md) | | `sentryTracingService` | [`SentryTracingService`](modules_sentry_sentry_tracing_service.SentryTracingService.md) | | `registrySettings` | `RegistrySettings` | | `ipfsService` | [`IPFSService`](modules_ipfs_ipfs_service.IPFSService.md) | +| `latestDidSyncRepository` | `Repository`<[`LatestDidSync`](modules_did_latestDidSync_entity.LatestDidSync.md)\> | +| `didSyncStatusRepository` | `Repository`<[`DidSyncStatusEntity`](modules_did_didSyncStatus_entity.DidSyncStatusEntity.md)\> | +| `dataSource` | `DataSource` | ## Methods diff --git a/docs/api/classes/modules_did_latestDidSync_entity.LatestDidSync.md b/docs/api/classes/modules_did_latestDidSync_entity.LatestDidSync.md new file mode 100644 index 000000000..282521338 --- /dev/null +++ b/docs/api/classes/modules_did_latestDidSync_entity.LatestDidSync.md @@ -0,0 +1,32 @@ +# Class: LatestDidSync + +[modules/did/latestDidSync.entity](../modules/modules_did_latestDidSync_entity.md).LatestDidSync + +## Table of contents + +### Constructors + +- [constructor](modules_did_latestDidSync_entity.LatestDidSync.md#constructor) + +### Properties + +- [block](modules_did_latestDidSync_entity.LatestDidSync.md#block) +- [createdDate](modules_did_latestDidSync_entity.LatestDidSync.md#createddate) + +## Constructors + +### constructor + +• **new LatestDidSync**() + +## Properties + +### block + +• **block**: `number` + +___ + +### createdDate + +• **createdDate**: `Date` diff --git a/docs/api/classes/modules_ipfs_ipfs_service.IPFSService.md b/docs/api/classes/modules_ipfs_ipfs_service.IPFSService.md index a1410f81a..4df04826f 100644 --- a/docs/api/classes/modules_ipfs_ipfs_service.IPFSService.md +++ b/docs/api/classes/modules_ipfs_ipfs_service.IPFSService.md @@ -18,15 +18,13 @@ ### constructor -• **new IPFSService**(`didStoreCluster`, `didStoreInfura`, `pinsQueue`, `logger`) +• **new IPFSService**(`didStoreInfura`, `logger`) #### Parameters | Name | Type | | :------ | :------ | -| `didStoreCluster` | `DidStore` | | `didStoreInfura` | `DidStore` | -| `pinsQueue` | `Queue`<[`PinClaimData`](../modules/modules_ipfs_ipfs_types.md#pinclaimdata)\> | | `logger` | [`Logger`](modules_logger_logger_service.Logger.md) | ## Methods diff --git a/docs/api/classes/modules_ipfs_pin_processor.PinProcessor.md b/docs/api/classes/modules_ipfs_pin_processor.PinProcessor.md deleted file mode 100644 index 4b0c0e85d..000000000 --- a/docs/api/classes/modules_ipfs_pin_processor.PinProcessor.md +++ /dev/null @@ -1,115 +0,0 @@ -# Class: PinProcessor - -[modules/ipfs/pin.processor](../modules/modules_ipfs_pin_processor.md).PinProcessor - -## Table of contents - -### Constructors - -- [constructor](modules_ipfs_pin_processor.PinProcessor.md#constructor) - -### Methods - -- [OnQueueWaiting](modules_ipfs_pin_processor.PinProcessor.md#onqueuewaiting) -- [onError](modules_ipfs_pin_processor.PinProcessor.md#onerror) -- [onFailed](modules_ipfs_pin_processor.PinProcessor.md#onfailed) -- [onStalled](modules_ipfs_pin_processor.PinProcessor.md#onstalled) -- [pin](modules_ipfs_pin_processor.PinProcessor.md#pin) - -## Constructors - -### constructor - -• **new PinProcessor**(`logger`, `didStoreCluster`, `didStoreInfura`) - -#### Parameters - -| Name | Type | -| :------ | :------ | -| `logger` | [`Logger`](modules_logger_logger_service.Logger.md) | -| `didStoreCluster` | `DidStore` | -| `didStoreInfura` | `DidStore` | - -## Methods - -### OnQueueWaiting - -▸ **OnQueueWaiting**(`jobId`): `Promise`<`void`\> - -#### Parameters - -| Name | Type | -| :------ | :------ | -| `jobId` | `number` | - -#### Returns - -`Promise`<`void`\> - -___ - -### onError - -▸ **onError**(`error`): `void` - -#### Parameters - -| Name | Type | -| :------ | :------ | -| `error` | `Error` | - -#### Returns - -`void` - -___ - -### onFailed - -▸ **onFailed**(`job`, `err`): `void` - -#### Parameters - -| Name | Type | -| :------ | :------ | -| `job` | `Job`<[`PinClaimData`](../modules/modules_ipfs_ipfs_types.md#pinclaimdata)\> | -| `err` | `Error` | - -#### Returns - -`void` - -___ - -### onStalled - -▸ **onStalled**(`job`): `void` - -#### Parameters - -| Name | Type | -| :------ | :------ | -| `job` | `Job`<[`PinClaimData`](../modules/modules_ipfs_ipfs_types.md#pinclaimdata)\> | - -#### Returns - -`void` - -___ - -### pin - -▸ **pin**(`job`): `Promise`<`void`\> - -This method migrates claims by retrieving from one DidStore and pinning to another -It was implemented for EW migration from Infura to EW hosted IPFS - -#### Parameters - -| Name | Type | -| :------ | :------ | -| `job` | `Job`<[`PinClaimData`](../modules/modules_ipfs_ipfs_types.md#pinclaimdata)\> | - -#### Returns - -`Promise`<`void`\> diff --git a/docs/api/enums/modules_did_did_types.DidSyncStatus.md b/docs/api/enums/modules_did_did_types.DidSyncStatus.md new file mode 100644 index 000000000..f7864674b --- /dev/null +++ b/docs/api/enums/modules_did_did_types.DidSyncStatus.md @@ -0,0 +1,22 @@ +# Enumeration: DidSyncStatus + +[modules/did/did.types](../modules/modules_did_did_types.md).DidSyncStatus + +## Table of contents + +### Enumeration Members + +- [Stale](modules_did_did_types.DidSyncStatus.md#stale) +- [Synced](modules_did_did_types.DidSyncStatus.md#synced) + +## Enumeration Members + +### Stale + +• **Stale** = ``1`` + +___ + +### Synced + +• **Synced** = ``0`` diff --git a/docs/api/modules.md b/docs/api/modules.md index bd1e1af96..1c0856ebe 100644 --- a/docs/api/modules.md +++ b/docs/api/modules.md @@ -91,6 +91,8 @@ - [modules/did/did.resolver](modules/modules_did_did_resolver.md) - [modules/did/did.service](modules/modules_did_did_service.md) - [modules/did/did.types](modules/modules_did_did_types.md) +- [modules/did/didSyncStatus.entity](modules/modules_did_didSyncStatus_entity.md) +- [modules/did/latestDidSync.entity](modules/modules_did_latestDidSync_entity.md) - [modules/ens/ens.module](modules/modules_ens_ens_module.md) - [modules/ens/ens.service](modules/modules_ens_ens_service.md) - [modules/interceptors/interceptors.module](modules/modules_interceptors_interceptors_module.md) @@ -101,7 +103,6 @@ - [modules/ipfs/ipfs.module](modules/modules_ipfs_ipfs_module.md) - [modules/ipfs/ipfs.service](modules/modules_ipfs_ipfs_service.md) - [modules/ipfs/ipfs.types](modules/modules_ipfs_ipfs_types.md) -- [modules/ipfs/pin.processor](modules/modules_ipfs_pin_processor.md) - [modules/logger/logger.module](modules/modules_logger_logger_module.md) - [modules/logger/logger.service](modules/modules_logger_logger_service.md) - [modules/nats/nats.module](modules/modules_nats_nats_module.md) diff --git a/docs/api/modules/modules_did_didSyncStatus_entity.md b/docs/api/modules/modules_did_didSyncStatus_entity.md new file mode 100644 index 000000000..a5546a35b --- /dev/null +++ b/docs/api/modules/modules_did_didSyncStatus_entity.md @@ -0,0 +1,7 @@ +# Module: modules/did/didSyncStatus.entity + +## Table of contents + +### Classes + +- [DidSyncStatusEntity](../classes/modules_did_didSyncStatus_entity.DidSyncStatusEntity.md) diff --git a/docs/api/modules/modules_did_did_types.md b/docs/api/modules/modules_did_did_types.md index 65b4899ba..c191549ec 100644 --- a/docs/api/modules/modules_did_did_types.md +++ b/docs/api/modules/modules_did_did_types.md @@ -2,10 +2,18 @@ ## Table of contents +### Enumerations + +- [DidSyncStatus](../enums/modules_did_did_types.DidSyncStatus.md) + ### Classes - [DID](../classes/modules_did_did_types.DID.md) +### Type Aliases + +- [UpdateDocumentJobData](modules_did_did_types.md#updatedocumentjobdata) + ### Variables - [ADD\_DID\_DOC\_JOB\_NAME](modules_did_did_types.md#add_did_doc_job_name) @@ -17,6 +25,18 @@ - [getDIDFromAddress](modules_did_did_types.md#getdidfromaddress) +## Type Aliases + +### UpdateDocumentJobData + +Ƭ **UpdateDocumentJobData**: `Object` + +#### Type declaration + +| Name | Type | +| :------ | :------ | +| `did` | `string` | + ## Variables ### ADD\_DID\_DOC\_JOB\_NAME diff --git a/docs/api/modules/modules_did_latestDidSync_entity.md b/docs/api/modules/modules_did_latestDidSync_entity.md new file mode 100644 index 000000000..022852a3a --- /dev/null +++ b/docs/api/modules/modules_did_latestDidSync_entity.md @@ -0,0 +1,7 @@ +# Module: modules/did/latestDidSync.entity + +## Table of contents + +### Classes + +- [LatestDidSync](../classes/modules_did_latestDidSync_entity.LatestDidSync.md) diff --git a/docs/api/modules/modules_ipfs_ipfs_types.md b/docs/api/modules/modules_ipfs_ipfs_types.md index c3dc9f9c3..19396b21e 100644 --- a/docs/api/modules/modules_ipfs_ipfs_types.md +++ b/docs/api/modules/modules_ipfs_ipfs_types.md @@ -6,14 +6,11 @@ - [IpfsClusterConfig](modules_ipfs_ipfs_types.md#ipfsclusterconfig) - [IpfsInfuraConfig](modules_ipfs_ipfs_types.md#ipfsinfuraconfig) -- [PinClaimData](modules_ipfs_ipfs_types.md#pinclaimdata) ### Variables - [IPFSClusterConfigToken](modules_ipfs_ipfs_types.md#ipfsclusterconfigtoken) - [IPFSInfuraConfigToken](modules_ipfs_ipfs_types.md#ipfsinfuraconfigtoken) -- [PIN\_CLAIM\_JOB\_NAME](modules_ipfs_ipfs_types.md#pin_claim_job_name) -- [PIN\_CLAIM\_QUEUE\_NAME](modules_ipfs_ipfs_types.md#pin_claim_queue_name) ## Type Aliases @@ -39,19 +36,6 @@ ___ | `protocol?` | `string` | | `url?` | `string` | -___ - -### PinClaimData - -Ƭ **PinClaimData**: `Object` - -#### Type declaration - -| Name | Type | -| :------ | :------ | -| `cid` | `string` | -| `claim?` | `string` | - ## Variables ### IPFSClusterConfigToken @@ -63,15 +47,3 @@ ___ ### IPFSInfuraConfigToken • `Const` **IPFSInfuraConfigToken**: typeof [`IPFSInfuraConfigToken`](modules_ipfs_ipfs_types.md#ipfsinfuraconfigtoken) - -___ - -### PIN\_CLAIM\_JOB\_NAME - -• `Const` **PIN\_CLAIM\_JOB\_NAME**: ``"pinning"`` - -___ - -### PIN\_CLAIM\_QUEUE\_NAME - -• `Const` **PIN\_CLAIM\_QUEUE\_NAME**: ``"pinClaimQueue"`` diff --git a/docs/api/modules/modules_ipfs_pin_processor.md b/docs/api/modules/modules_ipfs_pin_processor.md deleted file mode 100644 index fdbdef7d7..000000000 --- a/docs/api/modules/modules_ipfs_pin_processor.md +++ /dev/null @@ -1,7 +0,0 @@ -# Module: modules/ipfs/pin.processor - -## Table of contents - -### Classes - -- [PinProcessor](../classes/modules_ipfs_pin_processor.PinProcessor.md) diff --git a/e2e/ipfs/ipfs.testSuite.ts b/e2e/ipfs/ipfs.testSuite.ts index e0dd3cd2b..bc4a6b45b 100644 --- a/e2e/ipfs/ipfs.testSuite.ts +++ b/e2e/ipfs/ipfs.testSuite.ts @@ -1,5 +1,4 @@ import { HttpStatus } from '@nestjs/common'; -import { getQueueToken } from '@nestjs/bull'; import { Queue } from 'bull'; import request from 'supertest'; import { Connection, EntityManager, QueryRunner } from 'typeorm'; @@ -7,7 +6,6 @@ import { DidStore as DidStoreCluster } from 'didStoreCluster'; import { DidStore as DidStoreGateway } from 'didStoreInfura'; import { app } from '../app.e2e.spec'; import { randomUser } from '../utils'; -import { PIN_CLAIM_QUEUE_NAME } from '../../src/modules/ipfs/ipfs.types'; export const ipfsModuleTestSuite = () => { let queryRunner: QueryRunner; @@ -25,8 +23,6 @@ export const ipfsModuleTestSuite = () => { didStoreCluster = app.get(DidStoreCluster); didStoreInfura = app.get(DidStoreGateway); - pinsQueue = app.get(getQueueToken(PIN_CLAIM_QUEUE_NAME)); - await pinsQueue.empty(); const manager = app.get(EntityManager); const dbConnection = app.get(Connection); @@ -43,7 +39,8 @@ export const ipfsModuleTestSuite = () => { await queryRunner.release(); }); - it('save() should post claim in cluster', async () => { + // IPFS cluster is disabled + it.skip('save() should post claim in cluster', async () => { const claimData = { claimType: 'claim type', claimTypeVersion: 1, @@ -90,7 +87,8 @@ export const ipfsModuleTestSuite = () => { .expect(HttpStatus.NOT_FOUND); }); - it('if claim is not found in cluster, get() should save claim in cluster', async () => { + // IPFS cluster is disabled + it.skip('if claim is not found in cluster, get() should save claim in cluster', async () => { const didStoreClusterGet = jest.spyOn(didStoreCluster, 'get'); const didStoreInfuraGet = jest.spyOn(didStoreInfura, 'get'); diff --git a/src/migrations/1730905681141-LatestDidSync.ts b/src/migrations/1730905681141-LatestDidSync.ts new file mode 100644 index 000000000..91732fbd4 --- /dev/null +++ b/src/migrations/1730905681141-LatestDidSync.ts @@ -0,0 +1,15 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class LatestDidSync1730905681141 implements MigrationInterface { + name = 'LatestDidSync1730905681141'; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `CREATE TABLE "latest_did_sync" ("block" integer NOT NULL, "createdDate" TIMESTAMP NOT NULL DEFAULT now(), CONSTRAINT "PK_9e615062cb9a1c6b45598e74eae" PRIMARY KEY ("block"))` + ); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`DROP TABLE "latest_did_sync"`); + } +} diff --git a/src/migrations/1732050720611-AddDidSyncStatus.ts b/src/migrations/1732050720611-AddDidSyncStatus.ts new file mode 100644 index 000000000..ad557d584 --- /dev/null +++ b/src/migrations/1732050720611-AddDidSyncStatus.ts @@ -0,0 +1,30 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class AddDidSyncStatus1732050720611 implements MigrationInterface { + name = 'AddDidSyncStatus1732050720611'; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `CREATE TYPE "public"."did_sync_status_entity_status_enum" AS ENUM('0', '1')` + ); + await queryRunner.query( + `CREATE TABLE "did_sync_status_entity" ("id" SERIAL NOT NULL, "status" "public"."did_sync_status_entity_status_enum" NOT NULL DEFAULT '0', "created_date" TIMESTAMP NOT NULL DEFAULT now(), "document_id" character varying NOT NULL, CONSTRAINT "REL_100b8d489b25f9c448ec03e482" UNIQUE ("document_id"), CONSTRAINT "PK_e7f58d90249f1f87ddc5132cb11" PRIMARY KEY ("id"))` + ); + await queryRunner.query( + `ALTER TABLE "did_sync_status_entity" ADD CONSTRAINT "FK_100b8d489b25f9c448ec03e482d" FOREIGN KEY ("document_id") REFERENCES "did_document_entity"("id") ON DELETE NO ACTION ON UPDATE NO ACTION` + ); + await queryRunner.query( + `INSERT INTO did_sync_status_entity (document_id) SELECT id FROM did_document_entity` + ); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `ALTER TABLE "did_sync_status_entity" DROP CONSTRAINT "FK_100b8d489b25f9c448ec03e482d"` + ); + await queryRunner.query(`DROP TABLE "did_sync_status_entity"`); + await queryRunner.query( + `DROP TYPE "public"."did_sync_status_entity_status_enum"` + ); + } +} diff --git a/src/modules/did/did.module.ts b/src/modules/did/did.module.ts index d87c6e883..c46ef1f32 100644 --- a/src/modules/did/did.module.ts +++ b/src/modules/did/did.module.ts @@ -5,6 +5,7 @@ import { Methods } from '@ew-did-registry/did'; import { Provider } from '../../common/provider'; import { DIDController } from './did.controller'; import { DIDDocumentEntity } from './did.entity'; +import { LatestDidSync } from './latestDidSync.entity'; import { DIDProcessor } from './did.processor'; import { DIDResolver } from './did.resolver'; import { DIDService } from './did.service'; @@ -12,6 +13,7 @@ import { ethrReg } from '@ew-did-registry/did-ethr-resolver'; import { ConfigService } from '@nestjs/config'; import { BullModule } from '@nestjs/bull'; import { UPDATE_DOCUMENT_QUEUE_NAME } from './did.types'; +import { DidSyncStatusEntity } from './didSyncStatus.entity'; const RegistrySettingsProvider = { provide: 'RegistrySettings', @@ -29,7 +31,11 @@ const RegistrySettingsProvider = { BullModule.registerQueue({ name: UPDATE_DOCUMENT_QUEUE_NAME, }), - TypeOrmModule.forFeature([DIDDocumentEntity]), + TypeOrmModule.forFeature([ + DIDDocumentEntity, + LatestDidSync, + DidSyncStatusEntity, + ]), ], controllers: [DIDController], providers: [ @@ -41,4 +47,4 @@ const RegistrySettingsProvider = { ], exports: [DIDService, RegistrySettingsProvider], }) -export class DIDModule { } +export class DIDModule {} diff --git a/src/modules/did/did.processor.ts b/src/modules/did/did.processor.ts index e7d6a918f..7b7def719 100644 --- a/src/modules/did/did.processor.ts +++ b/src/modules/did/did.processor.ts @@ -5,7 +5,7 @@ import { OnQueueStalled, OnQueueWaiting, Process, - Processor + Processor, } from '@nestjs/bull'; import { ConfigService } from '@nestjs/config'; import { Job } from 'bull'; @@ -13,6 +13,7 @@ import { Logger } from '../logger/logger.service'; import { DIDService } from './did.service'; import { ADD_DID_DOC_JOB_NAME, + UpdateDocumentJobData, UPDATE_DID_DOC_JOB_NAME, UPDATE_DOCUMENT_QUEUE_NAME, } from './did.types'; @@ -22,7 +23,7 @@ export class DIDProcessor { constructor( private readonly didService: DIDService, private readonly logger: Logger, - private readonly configService: ConfigService, + private readonly configService: ConfigService ) { this.logger.setContext(DIDProcessor.name); } @@ -33,23 +34,23 @@ export class DIDProcessor { } @OnQueueActive() - onActive(job: Job) { - this.logger.debug(`Starting ${job.name} document ${job.data}`); + onActive(job: Job) { + this.logger.debug(`Starting ${job.name} document ${job.data.did}`); } @OnQueueStalled() - onStalled(job: Job) { - this.logger.debug(`Stalled ${job.name} document ${job.data}`); + onStalled(job: Job) { + this.logger.debug(`Stalled ${job.name} document ${job.data.did}`); } @OnQueueFailed() - onFailed(job: Job) { - this.logger.debug(`Failed ${job.name} document ${job.data}`); + onFailed(job: Job) { + this.logger.debug(`Failed ${job.name} document ${job.data.did}`); } @OnQueueWaiting() - async OnQueueWaiting(job: Job) { - this.logger.debug(`Waiting ${job.name} document ${job.data}`); + async OnQueueWaiting(jobId: number) { + this.logger.debug(`Waiting job ${jobId}`); } @Process(ADD_DID_DOC_JOB_NAME) @@ -58,11 +59,11 @@ export class DIDProcessor { } @Process(UPDATE_DID_DOC_JOB_NAME) - public async processDIDDocumentRefresh(job: Job) { + public async processDIDDocumentRefresh(job: Job) { if (this.configService.get('DID_SYNC_MODE_FULL')) { - await this.didService.addCachedDocument(job.data, true); + await this.didService.addCachedDocument(job.data.did, true); } else { - await this.didService.incrementalRefreshCachedDocument(job.data); + await this.didService.incrementalRefreshCachedDocument(job.data.did); } } } diff --git a/src/modules/did/did.service.spec.ts b/src/modules/did/did.service.spec.ts index f5e6899e8..12ff3bf71 100644 --- a/src/modules/did/did.service.spec.ts +++ b/src/modules/did/did.service.spec.ts @@ -18,6 +18,9 @@ import { SentryTracingService } from '../sentry/sentry-tracing.service'; import { EthereumDIDRegistry } from '../../ethers/EthereumDIDRegistry'; import { UPDATE_DOCUMENT_QUEUE_NAME } from './did.types'; import { IPFSService } from '../ipfs/ipfs.service'; +import { LatestDidSync } from './latestDidSync.entity'; +import { DidSyncStatusEntity } from './didSyncStatus.entity'; +import { DataSource } from 'typeorm'; const { formatBytes32String } = utils; @@ -66,6 +69,21 @@ jest.mock('@ew-did-registry/did-ethr-resolver', () => ({ }), })); +const dataSourceMockFactory = jest.fn(() => ({ + createQueryRunner: jest.fn().mockImplementation(() => ({ + connect: jest.fn(), + startTransaction: jest.fn(), + release: jest.fn(), + rollbackTransaction: jest.fn(), + commitTransaction: jest.fn(), + manager: { + save: jest.fn((_target, entity) => entity), + insert: jest.fn((_target, entity) => entity), + upsert: jest.fn((_target, entity, _conflict) => entity), + }, + })), +})); + describe('DidDocumentService', () => { let service: DIDService; let didRegistry: EthereumDIDRegistry; @@ -104,6 +122,14 @@ describe('DidDocumentService', () => { provide: getRepositoryToken(DIDDocumentEntity), useFactory: repositoryMockFactory, }, + { + provide: getRepositoryToken(LatestDidSync), + useFactory: repositoryMockFactory, + }, + { + provide: getRepositoryToken(DidSyncStatusEntity), + useFactory: repositoryMockFactory, + }, { provide: Provider, useValue: provider }, { provide: SentryTracingService, useValue: MockSentryTracing }, { @@ -116,6 +142,7 @@ describe('DidDocumentService', () => { inject: [ConfigService], }, { provide: IPFSService, useValue: MockObject }, + { provide: DataSource, useFactory: dataSourceMockFactory }, ], }).compile(); await module.init(); diff --git a/src/modules/did/did.service.ts b/src/modules/did/did.service.ts index e5d49224c..764f7ae6f 100644 --- a/src/modules/did/did.service.ts +++ b/src/modules/did/did.service.ts @@ -16,7 +16,7 @@ import { Transaction } from '@sentry/types'; import { isJWT } from 'class-validator'; import { firstValueFrom } from 'rxjs'; import { Queue } from 'bull'; -import { Repository } from 'typeorm'; +import { DataSource, Repository } from 'typeorm'; import jwt from 'jsonwebtoken'; import { BigNumber } from 'ethers'; import { @@ -35,6 +35,9 @@ import { EthereumDIDRegistry__factory } from '../../ethers/factories/EthereumDID import { EthereumDIDRegistry } from '../../ethers/EthereumDIDRegistry'; import { DID, + DidSyncStatus, + getDIDFromAddress, + UpdateDocumentJobData, UPDATE_DID_DOC_JOB_NAME, UPDATE_DOCUMENT_QUEUE_NAME, } from './did.types'; @@ -45,26 +48,35 @@ import { SentryTracingService } from '../sentry/sentry-tracing.service'; import { isVerifiableCredential } from '@ew-did-registry/credentials-interface'; import { IPFSService } from '../ipfs/ipfs.service'; import { inspect } from 'util'; +import { LatestDidSync } from './latestDidSync.entity'; +import { DidSyncStatusEntity } from './didSyncStatus.entity'; @Injectable() export class DIDService implements OnModuleInit, OnModuleDestroy { private readonly didRegistry: EthereumDIDRegistry; private readonly resolver: Resolver; private readonly JOBS_CLEANUP_DELAY = 1000; + private readonly MAX_EVENTS_QUERY_INTERVAL = 1000000; + private readonly MAX_SYNC_DOCUMENTS = 1000; constructor( private readonly config: ConfigService, private readonly schedulerRegistry: SchedulerRegistry, private readonly httpService: HttpService, @InjectQueue(UPDATE_DOCUMENT_QUEUE_NAME) - private readonly didQueue: Queue, + private readonly didQueue: Queue, private readonly logger: Logger, @InjectRepository(DIDDocumentEntity) private readonly didRepository: Repository, private readonly provider: Provider, private readonly sentryTracingService: SentryTracingService, @Inject('RegistrySettings') registrySettings: RegistrySettings, - private readonly ipfsService: IPFSService + private readonly ipfsService: IPFSService, + @InjectRepository(LatestDidSync) + private readonly latestDidSyncRepository: Repository, + @InjectRepository(DidSyncStatusEntity) + private readonly didSyncStatusRepository: Repository, + private dataSource: DataSource ) { this.logger.setContext(DIDService.name); @@ -215,7 +227,26 @@ export class DIDService implements OnModuleInit, OnModuleDestroy { logs: JSON.stringify(logs), }); - return this.didRepository.save(updatedEntity); + const queryRunner = this.dataSource.createQueryRunner(); + await queryRunner.connect(); + await queryRunner.startTransaction(); + try { + const updated = await queryRunner.manager.save( + DIDDocumentEntity, + updatedEntity + ); + await queryRunner.manager.insert(DidSyncStatusEntity, { + document: updated, + status: DidSyncStatus.Synced, + }); + await queryRunner.commitTransaction(); + this.logger.debug(`Document ${did} was synchronized`); + return updated; + } catch (_) { + await queryRunner.rollbackTransaction(); + } finally { + await queryRunner.release(); + } } catch (err) { this.logger.error(err); } finally { @@ -243,7 +274,7 @@ export class DIDService implements OnModuleInit, OnModuleDestroy { }); try { - this.logger.info(`refreshing cached document for did: ${did}`); + this.logger.info(`Refreshing cached document for did: ${did}`); let span = transaction?.startChild({ op: 'find_did_document', description: 'Find DID Document', @@ -288,7 +319,31 @@ export class DIDService implements OnModuleInit, OnModuleDestroy { logs: JSON.stringify(logs), }); - return this.didRepository.save(updatedEntity); + const queryRunner = this.dataSource.createQueryRunner(); + await queryRunner.connect(); + await queryRunner.startTransaction(); + try { + const updated = await queryRunner.manager.save( + DIDDocumentEntity, + updatedEntity + ); + await queryRunner.manager.upsert( + DidSyncStatusEntity, + { + document: updated, + status: DidSyncStatus.Synced, + }, + ['document'] + ); + await queryRunner.commitTransaction(); + this.logger.debug(`Document ${did} was synchronized`); + return updated; + } catch (e) { + await queryRunner.rollbackTransaction(); + this.logger.error(`Failed to synchronize DID ${did}: ${e}`); + } finally { + await queryRunner.release(); + } } catch (err) { this.logger.error(err); } finally { @@ -359,10 +414,19 @@ export class DIDService implements OnModuleInit, OnModuleDestroy { } private async syncDocuments() { - this.logger.debug(`Beginning sync of DID Documents`); - const cachedDIDs = await this.didRepository.find({ select: ['id'] }); - cachedDIDs.forEach(async (did) => { - await this.pinDocument(did.id); + await this.markStaleDocuments(); + + const staleDIDs = ( + await this.didSyncStatusRepository.find({ + where: { status: DidSyncStatus.Stale }, + relations: { document: true }, + take: this.MAX_SYNC_DOCUMENTS, + }) + ).map((status) => status.document.id); + this.logger.debug(`Synchronizing ${staleDIDs.length} documents`); + staleDIDs.forEach(async (did) => { + this.logger.debug(`Synchronizing DID ${did}`); + await this.pinDocument(did); }); } @@ -493,7 +557,8 @@ export class DIDService implements OnModuleInit, OnModuleDestroy { private async pinDocument(did: string): Promise { try { - await this.didQueue.add(UPDATE_DID_DOC_JOB_NAME, did, { jobId: did }); + // Specifying job id to avoid adding job for the same document + await this.didQueue.add(UPDATE_DID_DOC_JOB_NAME, { did }, { jobId: did }); } catch (e) { this.logger.warn( `Error to add DID synchronization job for document ${did}: ${e}` @@ -502,4 +567,102 @@ export class DIDService implements OnModuleInit, OnModuleDestroy { this.logger.debug(inspect(jobsCounts, { depth: 2, colors: true })); } } + + /** + * Finds DIDs changed in given block interval. If fetching of DIDs fails at some block, then further fetching stops. + * + * Returns last successfully synced block + */ + private async getChangedIdentities( + fromBlock: number, + topBlock: number + ): Promise<{ changedIdentities: string[]; syncedBlock: number }> { + const didEventFilters = [ + this.didRegistry.filters.DIDAttributeChanged(null), + this.didRegistry.filters.DIDDelegateChanged(null), + this.didRegistry.filters.DIDOwnerChanged(null), + ]; + const events = []; + let syncedBlock = fromBlock; + while (fromBlock < topBlock) { + const toBlock = Math.min( + topBlock, + fromBlock + this.MAX_EVENTS_QUERY_INTERVAL + ); + try { + const intervalEvents = ( + await Promise.all( + didEventFilters.map((filter) => + this.didRegistry.queryFilter(filter, fromBlock, toBlock) + ) + ) + ).flat(); + this.logger.debug( + `Fetched ${intervalEvents.length} DID events from interval [${fromBlock}, ${toBlock}]` + ); + events.push(...intervalEvents); + syncedBlock = toBlock; + fromBlock = toBlock; + } catch (e) { + this.logger.error( + `Failed to fetch DID events from interval [${fromBlock}, ${toBlock}]: ${e.message}` + ); + break; + } + } + + this.logger.debug(`Update document events count ${events.length}`); + const changedIdentities = events.map((event) => { + return event.args.identity; + }); + return { + // Deduplicate identities if they appear in multiple events + changedIdentities: [...new Set(changedIdentities).values()], + syncedBlock, + }; + } + + /** + * Finds documents changed since last synchronization and sets their sync status to `Stale` + */ + private async markStaleDocuments() { + const syncs = await this.latestDidSyncRepository.find({ + order: { createdDate: 'DESC' }, + }); + const fromBlock = syncs.length > 0 ? syncs[0].block : 0; + const topBlock = await this.provider.getBlockNumber(); + const { changedIdentities, syncedBlock } = await this.getChangedIdentities( + fromBlock, + topBlock + ); + this.logger.debug( + `Fetched DID update events from block ${fromBlock} to block ${syncedBlock}` + ); + const changedDIDs = changedIdentities.map(getDIDFromAddress); + if (changedDIDs.length > 0) { + const changedCachedDIDs: { document_id: string }[] = + await this.didRepository + .createQueryBuilder('document') + .select('document.id') + .where('document.id IN (:...changedDIDs)', { changedDIDs }) + .execute(); + + const { identifiers } = await this.didSyncStatusRepository + .createQueryBuilder() + .insert() + .into(DidSyncStatusEntity) + .values( + changedCachedDIDs.map(({ document_id }) => ({ + document: { id: document_id }, + status: DidSyncStatus.Stale, + })) + ) + .orUpdate(['status'], ['document_id']) + .execute(); + + this.logger.debug(`Marked ${identifiers.length} stale documents`); + } + + await this.latestDidSyncRepository.save({ block: syncedBlock }); + } } diff --git a/src/modules/did/did.types.ts b/src/modules/did/did.types.ts index e87493a34..94d982bc9 100644 --- a/src/modules/did/did.types.ts +++ b/src/modules/did/did.types.ts @@ -110,3 +110,12 @@ export const getDIDFromAddress = (address: string) => export const ADD_DID_DOC_JOB_NAME = 'adding'; export const UPDATE_DID_DOC_JOB_NAME = 'refreshing'; export const UPDATE_DOCUMENT_QUEUE_NAME = 'updateDocumentQueue'; + +export type UpdateDocumentJobData = { + did: string; +}; + +export enum DidSyncStatus { + Synced, + Stale, +} diff --git a/src/modules/did/didSyncStatus.entity.ts b/src/modules/did/didSyncStatus.entity.ts new file mode 100644 index 000000000..4ecff53d2 --- /dev/null +++ b/src/modules/did/didSyncStatus.entity.ts @@ -0,0 +1,28 @@ +import { + Column, + CreateDateColumn, + Entity, + JoinColumn, + OneToOne, + PrimaryGeneratedColumn, +} from 'typeorm'; +import { DIDDocumentEntity } from './did.entity'; +import { DidSyncStatus } from './did.types'; + +@Entity() +export class DidSyncStatusEntity { + @PrimaryGeneratedColumn() + id: number; + + @OneToOne(() => DIDDocumentEntity, (document) => document.id, { + nullable: false, + }) + @JoinColumn({ name: 'document_id' }) + document: DIDDocumentEntity; + + @Column({ enum: DidSyncStatus, type: 'enum', default: DidSyncStatus.Synced }) + status: DidSyncStatus; + + @CreateDateColumn({ name: 'created_date' }) + createdDate: Date; +} diff --git a/src/modules/did/latestDidSync.entity.ts b/src/modules/did/latestDidSync.entity.ts new file mode 100644 index 000000000..382c27b08 --- /dev/null +++ b/src/modules/did/latestDidSync.entity.ts @@ -0,0 +1,10 @@ +import { CreateDateColumn, Entity, PrimaryColumn } from 'typeorm'; + +@Entity() +export class LatestDidSync { + @PrimaryColumn() + block: number; + + @CreateDateColumn() + createdDate: Date; +} diff --git a/src/modules/ipfs/ipfs.types.ts b/src/modules/ipfs/ipfs.types.ts index 7dbfc1667..5177d1b77 100644 --- a/src/modules/ipfs/ipfs.types.ts +++ b/src/modules/ipfs/ipfs.types.ts @@ -16,11 +16,3 @@ export type IpfsInfuraConfig = { export const IPFSInfuraConfigToken = Symbol.for('IPFSInfuraConfigToken'); export const IPFSClusterConfigToken = Symbol.for('IPFSClusterConfigToken'); - -export const PIN_CLAIM_JOB_NAME = 'pinning'; -export const PIN_CLAIM_QUEUE_NAME = 'pinClaimQueue'; - -export type PinClaimData = { - cid: string; - claim?: string; -}; diff --git a/src/modules/logger/logger.service.ts b/src/modules/logger/logger.service.ts index e19c5293b..e2ac2d61b 100644 --- a/src/modules/logger/logger.service.ts +++ b/src/modules/logger/logger.service.ts @@ -26,6 +26,12 @@ export class Logger extends NestLogger implements LoggerService { ) { super(); this.redactor = new SyncRedactor({ + builtInRedactors: { + digits: { + enabled: false, + }, + zipcode: { enabled: false }, + }, customRedactors: { before: [ {