Merge pull request #1451 from multiversx/development

Development

Author: dragos-rebegea

.github/workflows/chain-simulator-e2e-tests.yml

@@ -0,0 +1,65 @@
+name: Chain simulator e2e tests workflow
+
+on:
+  push:
+    branches: [main, development]
+  pull_request:
+    branches: [main, development]
+
+jobs:
+  test-chainsimulator-e2e:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        node-version: [18.x]
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node-version }}
+          cache: 'npm'
+
+      - name: Build and start chain simulator
+        run: npm run start-chain-simulator
+
+      - name: Wait for services to be ready
+        run: |
+          echo "Waiting for services to be healthy..."
+          docker ps
+          docker logs chainsimulator
+          sleep 20 # Wait for 20 seconds to ensure services are up
+
+      - name: Print docker containers
+        run: docker ps
+
+      - name: Start API Docker containers
+        run: |
+          cd .
+          docker compose up -d
+
+      - name: Wait for API docker containers to start
+        run: |
+          sleep 20
+          docker ps
+
+      - run: npm ci
+      - run: npm run init
+
+      - name: Start API
+        run: |
+          npm run start:mainnet:e2e &
+          sleep 10 # Wait a little more to ensure the API is fully up
+      
+      - name: Prepare Test Data
+        run: npm run prepare:test-data
+
+      - name: Run e2e tests
+        run: npm run test:cs-e2e
+
+      - name: Stop API after tests
+        run: |
+          echo "Stopping the API..."
+          kill $(lsof -t -i:3001)

.github/workflows/load-tests.yml

@@ -58,7 +58,7 @@ jobs:
       run: k6 run ./k6/script.js
 
     - name: Upload result file for base branch
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         name: base-results
         path: k6/output/summary.json
@@ -119,7 +119,7 @@ jobs:
       run: k6 run ./k6/script.js
 
     - name: Upload result file for head branch
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         name: head-results
         path: k6/output/summary.json
@@ -135,7 +135,7 @@ jobs:
       - uses: actions/checkout@v2
 
       - name: Download all artifacts
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           path: artifacts
 
@@ -159,7 +159,7 @@ jobs:
             head: ${{ github.event.pull_request.head.sha }}
 
       - name: Upload the report markdown
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: github.event_name == 'pull_request'
         with:
           name: report-markdown

.gitignore

@@ -41,4 +41,7 @@ lerna-debug.log*
 
 /src/plugins
 
-.env
+.env
+
+# CS Environment variables
+src/test/chain-simulator/config/.env

config/config.e2e.mainnet.yaml

@@ -2,9 +2,51 @@ network: 'mainnet'
 metaChainShardId: 4294967295
 api:
   public: true
+  publicPort: 3001
   private: true
-  graphql: true
+  privatePort: 4001
+  websocket: true
+cron:
+  cacheWarmer: true
+  fastWarm: false
+  queueWorker: true
+  elasticUpdater: false
+flags:
+  useRequestCaching: true
+  useKeepAliveAgent: true
+  useTracing: false
+  useRequestLogging: false
+  useVmQueryTracing: false
+  processNfts: true
+  collectionPropertiesFromGateway: false
 features:
+  eventsNotifier:
+    enabled: false
+    port: 5674
+    url: 'amqp://guest:guest@127.0.0.1:5672'
+    exchange: 'all_events'
+    queue: 'api-process-logs-and-events'
+  guestCaching:
+    enabled: false
+    hitsThreshold: 100
+    ttl: 12
+  transactionPool:
+    enabled: true
+  transactionPoolWarmer:
+    enabled: true
+    cronExpression: '*/5 * * * * *'
+    ttlInSeconds: 60
+  updateCollectionExtraDetails:
+    enabled: false
+  updateAccountExtraDetails:
+    enabled: false
+    transfersLast24hUrl: 'https://tools.multiversx.com/growth-api/explorer/widgets/most-used/applications'
+  marketplace:
+    enabled: false
+    serviceUrl: 'https://nfts-graph.multiversx.com/graphql'
+  exchange:
+    enabled: false
+    serviceUrl: 'https://graph.xexchange.com/graphql'
   dataApi:
     enabled: false
     serviceUrl: 'https://data-api.multiversx.com'
@@ -13,35 +55,93 @@ features:
     maxExpirySeconds: 86400
     acceptedOrigins:
       - ''
+    admins:
+      - ''
+    jwtSecret: ''
+  stakingV4:
+    enabled: false
+    cronExpression: '*/5 * * * * *'
+    activationEpoch: 1391
+  nodeEpochsLeft:
+    enabled: false
+  transactionProcessor:
+    enabled: false
+    maxLookBehind: 1000
+  transactionCompleted:
+    enabled: false
+    maxLookBehind: 1000
+    logLevel: 'Error'
+  transactionBatch:
+    enabled: false
+    maxLookBehind: 1000
   deepHistory:
     enabled: false
     url: ''
-cron:
-  transactionProcessor: false
-  transactionProcessorMaxLookBehind: 1000
-  cacheWarmer: false
-flags:
-  useRequestCaching: true
-  useKeepAliveAgent: true
-  useTracing: false
-  collectionPropertiesFromGateway: false
+  statusChecker:
+    enabled: false
+    thresholds:
+      tokens: 1000
+      nodes: 5000
+      providers: 150
+      tokenSupplyCount: 100
+      tokenAssets: 100
+      tokenAccounts: 1000
+      tokenTransactions: 1000
+      nodeValidators: 3260
+  nftScamInfo:
+    enabled: false
+  processNfts:
+    enabled: false
+    nftQueueName: 'api-process-nfts'
+    deadLetterQueueName: 'api-process-nfts-dlq'
+  tps:
+    enabled: false
+    maxLookBehindNonces: 100
+  nodesFetch:
+    enabled: false
+    serviceUrl: 'https://api.multiversx.com'
+  tokensFetch:
+    enabled: false
+    serviceUrl: 'https://api.multiversx.com'
+  providersFetch:
+    enabled: false
+    serviceUrl: 'https://api.multiversx.com'
+  assetsFetch:
+    enabled: false
+    assetesUrl: 'https://tools.multiversx.com/assets-cdn'
+image:
+  width: 600
+  height: 600
+  type: 'png'
+aws:
+  s3KeyId: ''
+  s3Secret: ''
+  s3Bucket: 'media.elrond.com'
+  s3Region: ''
 urls:
-  self: 'https://api.multiversx.com'
+  self: 'http://localhost:3001'
   elastic:
-    - 'https://index.multiversx.com'
+    - 'http://localhost:9200'
   gateway:
-    - 'https://gateway.multiversx.com'
+    - 'http://localhost:8085'
   verifier: 'https://play-api.multiversx.com'
   redis: '127.0.0.1'
   rabbitmq: 'amqp://127.0.0.1:5672'
-  providers: 'https://internal-delegation-api.multiversx.com/providers'
+  providers: 'https://delegation-api.multiversx.com/providers'
   delegation: 'https://delegation-api.multiversx.com'
   media: 'https://media.elrond.com'
   nftThumbnails: 'https://media.elrond.com/nfts/thumbnail'
+  tmp: '/tmp'
+  ipfs: 'https://ipfs.io/ipfs'
+  socket: 'socket-api-fra.multiversx.com'
   maiarId: 'https://id-api.multiversx.com'
+indexer:
+  type: 'elastic'
+  maxPagination: 10000
 database:
   enabled: false
   url: 'mongodb://127.0.0.1:27017/api?authSource=admin'
+  type: 'mysql'
   host: 'localhost'
   port: 3306
   username: 'root'
@@ -50,7 +150,8 @@ database:
 caching:
   cacheTtl: 6
   processTtl: 600
-  poolLimit: 10
+  poolLimit: 50
+  cacheDuration: 3
 keepAliveTimeout:
   downstream: 61000
   upstream: 60000
@@ -69,14 +170,6 @@ inflation:
   - 1130177
   - 924690
   - 719203
-security:
-  admins:
-  jwtSecret:
-test:
-  mockKeybases: false
-  mockNodes: false
-  mockTokens: false
-  mockPath: './src/test/mocks/'
-transaction-action:
-  mex:
-    microServiceUrl: 'https://graph.xexchange.com/graphql'
+nftProcess:
+  parallelism: 1
+  maxRetries: 3

package-lock.json

@@ -28,6 +28,9 @@
     "start:mainnet": "npm run copy-mainnet-config & nest start",
     "start:mainnet:watch": "npm run copy-mainnet-config & nest start --watch",
     "start:mainnet:debug": "npm run copy-mainnet-config & nest start --watch --debug",
+    "start:mainnet:e2e": "npm run copy-e2e-mainnet-config & nest start",
+    "start:mainnet:e2e:watch": "npm run copy-e2e-mainnet-config & nest start --watch",
+    "start:mainnet:e2e:debug": "npm run copy-e2e-mainnet-config & nest start --watch --debug",
     "lint": "eslint \"{src,apps,libs,test}/**/*.ts\"",
     "lint:fix": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix",
     "test": "npm run copy-mainnet-config & nest build & jest",
@@ -41,6 +44,7 @@
     "test:e2e:local": "npm run test:e2e:warm & npm run test:e2e",
     "test:api": "jest --config ./src/test/jest-api.json --runInBand --detectOpenHandles --forceExit",
     "test:graph": "jest --config ./src/test/jest-graph-spec.json --runInBand --detectOpenHandles --forceExit",
+    "test:cs-e2e": "jest --config ./src/test/jest-chain-simulator.json --runInBand --detectOpenHandles --forceExit",
     "init": "run-script-os",
     "copy-mainnet-config": "run-script-os",
     "copy-testnet-config": "run-script-os",
@@ -73,7 +77,10 @@
     "copy-e2e-mainnet-config:nix": "cp ./config/config.e2e.mainnet.yaml ./config/config.yaml",
     "copy-e2e-mocked-mainnet-config:nix": "cp ./config/config.e2e-mocked.mainnet.yaml ./config/config.yaml",
     "copy-e2e-mainnet-config:windows": "copy .\\config\\config.e2e.mainnet.yaml .\\config\\config.yaml",
-    "copy-e2e-mocked-mainnet-config:windows": "copy .\\config\\config.e2e-mocked.mainnet.yaml .\\config\\config.yaml"
+    "copy-e2e-mocked-mainnet-config:windows": "copy .\\config\\config.e2e-mocked.mainnet.yaml .\\config\\config.yaml",
+    "start-chain-simulator": "docker compose -f \"src/test/chain-simulator/docker/docker-compose.yml\" up -d --build",
+    "stop-chain-simulator": "docker compose -f \"src/test/chain-simulator/docker/docker-compose.yml\" down",
+    "prepare:test-data": "ts-node src/test/chain-simulator/utils/prepare-test-data.ts"
   },
   "dependencies": {
     "@aws-sdk/client-s3": "^3.54.0",
@@ -149,6 +156,7 @@
   "devDependencies": {
     "@automock/adapters.nestjs": "^2.1.0",
     "@automock/jest": "^2.1.0",
+    "@jest/test-sequencer": "^29.7.0",
     "@nestjs/cli": "10.1.17",
     "@nestjs/schematics": "10.0.2",
     "@nestjs/testing": "10.2.4",
@@ -157,7 +165,7 @@
     "@types/crypto-js": "^4.0.2",
     "@types/express": "^4.17.13",
     "@types/fluent-ffmpeg": "^2.1.20",
-    "@types/jest": "29.5.0",
+    "@types/jest": "^29.5.0",
     "@types/js-yaml": "^4.0.5",
     "@types/json-diff": "^0.7.0",
     "@types/jsonwebtoken": "^8.5.8",
@@ -176,7 +184,7 @@
     "prettier": "^2.5.1",
     "run-script-os": "^1.1.6",
     "supertest": "^6.2.2",
-    "ts-jest": "29.0.5",
+    "ts-jest": "^29.0.5",
     "ts-loader": "9.4.2",
     "ts-node": "10.7.0",
     "tsconfig-paths": "^4.0.0",

package.json

@@ -16,6 +16,9 @@ export class GraphQlService {
     const exchangeServiceUrl = this.apiConfigService.getExchangeServiceUrlMandatory();
     const graphqlClient = new GraphQLClient(exchangeServiceUrl, {
       fetch: this.createFetchWithTimeout(60_000),
+      headers: {
+        'origin': this.apiConfigService.getSelfUrl(),
+      },
     });
 
     try {