qa-tests: complete erigon vs geth benchmark workflow

.github/workflows/qa-rpc-performance-comparison-tests.yml

@@ -11,11 +11,12 @@ concurrency:
 env:
   CHAIN: mainnet
   ERIGON_QA_PATH: /home/qarunner/erigon-qa
+  RPC_PAST_TEST_DIR: /opt/rpc-past-tests
 
 jobs:
   setup:
     name: Setup benchmark run
-    runs-on: ubuntu-latest
+    runs-on: [self-hosted, qa]
     permissions:
       contents: read
     outputs:
@@ -61,17 +62,25 @@ jobs:
 
       - name: Prepare Redis database
         shell: bash
-        working-directory: ${{ env.ERIGON_QA_PATH }}/test_system/qa-tests/tip-tracking/rpc-tests
+        working-directory: ${{ env.ERIGON_QA_PATH }}/test_system/qa-tests/rpc-tests
         run: |
           set -euo pipefail
-          python redis_client.py --key ${{ steps.vars.outputs.erigon_redis_key }} --write "starting"
-          python redis_client.py --key ${{ steps.vars.outputs.geth_redis_key }} --write "starting"
-
-  test_erigon:
-    name: Benchmark Erigon
-    needs: [ setup ]
-    runs-on: [ self-hosted, qa, Ethereum, rpc-latest-erigon ]
-    timeout-minutes: 120
+          python3 redis_client.py --key ${{ steps.vars.outputs.erigon_redis_key }} --write "created"
+          python3 redis_client.py --key ${{ steps.vars.outputs.geth_redis_key }} --write "created"
+
+  test:
+    name: Benchmark ${{ matrix.client }}
+    needs: [setup]
+    strategy:
+      fail-fast: true
+      matrix:
+        include:
+          - client: erigon
+            runs-on: [self-hosted, qa, Ethereum, rpc-latest-erigon]
+          - client: geth
+            runs-on: [self-hosted, qa, Ethereum, rpc-latest-geth]
+    runs-on: ${{ matrix.runs-on }}
+    timeout-minutes: 480
     permissions:
       contents: read
     env:
@@ -81,68 +90,281 @@ jobs:
       VEGETA_DURATION: ${{ needs.setup.outputs.vegeta_duration }}
       ERIGON_REDIS_KEY: ${{ needs.setup.outputs.erigon_redis_key }}
       GETH_REDIS_KEY: ${{ needs.setup.outputs.geth_redis_key }}
+      ERIGON_REFERENCE_DIR: /opt/erigon-versions/reference-version
+      ERIGON_REFERENCE_DATA_DIR: /opt/erigon-versions/reference-version/datadir
+      ERIGON_TESTBED_DATA_DIR: /opt/erigon-testbed/datadir
+      GETH_INSTALL_DIR: /opt/go-ethereum
+      CLIENT: ${{ matrix.client }}
     steps:
       - name: Checkout
         uses: actions/checkout@v6
 
       - name: Sanity check Redis config
         shell: bash
-        working-directory: ${{ env.ERIGON_QA_PATH }}/test_system/qa-tests/tip-tracking/rpc-tests
+        working-directory: ${{ env.ERIGON_QA_PATH }}/test_system/qa-tests/rpc-tests
         run: |
           set -euo pipefail
-          python redis_client.py --key ${{ needs.setup.outputs.erigon_redis_key }} --read
+          python3 redis_client.py --key ${{ matrix.client == 'erigon' && needs.setup.outputs.erigon_redis_key || needs.setup.outputs.geth_redis_key }} --read
 
-      - name: Run Erigon benchmark (placeholder)
-        shell: bash
+      - name: Checkout RPC Tests Repository
         run: |
-          set -euo pipefail
-          echo "TODO: Build/run Erigon, monitor sync, publish status to Redis, wait barrier, run Vegeta."
-          echo "Context:"
-          echo "  RUN_ID=${RUN_ID}"
-          echo "  REDIS_KEY_PREFIX=${REDIS_KEY_PREFIX}"
-          echo "  NETWORK=${NETWORK}"
-          echo "  VEGETA_DURATION=${VEGETA_DURATION}"
-          echo "  ERIGON_REDIS_KEY=${ERIGON_REDIS_KEY}"
-          echo "  GETH_REDIS_KEY=${GETH_REDIS_KEY}"
-          # Example placeholders for later:
-          # python3 scripts/bench_client.py --client erigon --redis-prefix "REDIS_KEY_PREFIX" --network "$NETWORK"
-
-  test_geth:
-    name: Benchmark Geth
-    needs: [ setup ]
-    runs-on: [ self-hosted, qa, Ethereum, rpc-latest-geth ]
-    timeout-minutes: 120
-    permissions:
-      contents: read
-    env:
-      RUN_ID: ${{ needs.setup.outputs.run_id }}
-      REDIS_KEY_PREFIX: ${{ needs.setup.outputs.redis_key_prefix }}
-      NETWORK: ${{ needs.setup.outputs.network }}
-      VEGETA_DURATION: ${{ needs.setup.outputs.vegeta_duration }}
-      ERIGON_REDIS_KEY: ${{ needs.setup.outputs.erigon_redis_key }}
-      GETH_REDIS_KEY: ${{ needs.setup.outputs.geth_redis_key }}
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v6
+          rm -rf ${{runner.workspace}}/rpc-tests
+          git -c advice.detachedHead=false clone --depth 1 --branch v1.78.0 https://github.com/erigontech/rpc-tests ${{runner.workspace}}/rpc-tests
+          cd ${{runner.workspace}}/rpc-tests
 
-      - name: Sanity check Redis config
-        shell: bash
-        working-directory: ${{ env.ERIGON_QA_PATH }}/test_system/qa-tests/tip-tracking/rpc-tests
+      - name: Clean Erigon Build Directory
+        if: matrix.client == 'erigon'
         run: |
-          set -euo pipefail
-          python redis_client.py --key ${{ needs.setup.outputs.geth_redis_key }} --read
+          make clean
+
+      - name: Build Erigon
+        if: matrix.client == 'erigon'
+        run: |
+          make erigon integration
+        working-directory: ${{ github.workspace }}
+
+      - name: Pause the Erigon instance dedicated to db maintenance
+        if: matrix.client == 'erigon'
+        run: |
+          python3 $ERIGON_QA_PATH/test_system/db-producer/pause_production.py || true
+
+      - name: Save Erigon datadir Directory
+        if: matrix.client == 'erigon'
+        id: save_datadir_step
+        run: |
+          rm -rf $ERIGON_TESTBED_DATA_DIR || true
+
+          echo "Mirror datadir"
+          if ! "$GITHUB_WORKSPACE/cmd/scripts/mirror-datadir.sh" "$ERIGON_REFERENCE_DATA_DIR" "$ERIGON_TESTBED_DATA_DIR" > /dev/null 2>&1; then
+            echo "Failed to mirror datadir from $ERIGON_REFERENCE_DATA_DIR to $ERIGON_TESTBED_DATA_DIR"
+            exit 1
+          fi
+          echo "datadir_saved=true" >> $GITHUB_OUTPUT
+
+      - name: Run Migrations
+        if: matrix.client == 'erigon'
+        working-directory: ${{ github.workspace }}/build/bin
+        run: |
+          echo "Running migrations on datadir..."
+          ./integration run_migrations --datadir $ERIGON_TESTBED_DATA_DIR --chain $CHAIN
 
-      - name: Run Geth benchmark (placeholder)
+      - name: Run Erigon
+        if: matrix.client == 'erigon'
+        id: erigon_running_step
+        working-directory: ${{ github.workspace }}/build/bin
+        run: |
+          set +e # Disable exit on error
+          echo "Starting Erigon..."
+
+          ./erigon --prune.mode=minimal --datadir $ERIGON_TESTBED_DATA_DIR --http.api admin,debug,eth,parity,erigon,trace,web3,txpool,ots,net --ws > erigon.log 2>&1 &
+
+          ERIGON_PID=$!
+          echo "ERIGON_PID=$ERIGON_PID" >> $GITHUB_ENV
+          echo "erigon_started=true" >> $GITHUB_OUTPUT
+
+          sleep 5
+          tail erigon.log
+
+          if ! kill -0 "$ERIGON_PID" 2>/dev/null; then
+            echo "Erigon failed to start"
+            echo "::error::Error detected during tests: Erigon failed to start"
+            exit 1
+          fi
+          echo "Erigon started"
+
+      - name: Wait for port 8545 to be opened
+        run: |
+          for i in {1..30}; do
+            if nc -z localhost 8545; then
+              echo "Port 8545 is open"
+              break
+            fi
+            echo "Waiting for port 8545 to open..."
+            sleep 10
+          done
+          if ! nc -z localhost 8545; then
+            echo "Port 8545 did not open in time"
+            echo "::error::Error detected during tests: Port 8545 did not open in time"
+            exit 1
+          fi
+
+      - name: Find a consensus on a common time to run the benchmark
         shell: bash
+        working-directory: ${{ env.ERIGON_QA_PATH }}/test_system/qa-tests/rpc-tests
         run: |
           set -euo pipefail
-          echo "TODO: Monitor Geth sync, publish status to Redis, wait barrier, run Vegeta."
+          echo "Monitor sync, publish status to Redis, wait barrier, run Vegeta."
           echo "Context:"
           echo "  RUN_ID=${RUN_ID}"
           echo "  REDIS_KEY_PREFIX=${REDIS_KEY_PREFIX}"
           echo "  NETWORK=${NETWORK}"
           echo "  VEGETA_DURATION=${VEGETA_DURATION}"
           echo "  ERIGON_REDIS_KEY=${ERIGON_REDIS_KEY}"
           echo "  GETH_REDIS_KEY=${GETH_REDIS_KEY}"
-          # Example placeholders for later:
-          # python3 scripts/bench_client.py --client geth --redis-prefix "REDIS_KEY_PREFIX" --network "$NETWORK"
+          # Wait for an agreement between Erigon and Geth
+          python3 coordinated_start.py --client "$CLIENT" --run-id "$RUN_ID" --chain "$NETWORK"
+
+      - name: Run RPC Performance Tests
+        id: test_step
+        run: |
+          set +e # Disable exit on error
+
+          failed_test=0
+
+          if [ "${{ matrix.client }}" == "erigon" ]; then
+            commit=$(git -C ${{runner.workspace}}/erigon rev-parse --short HEAD)
+          else
+            commit=$(git -C $GETH_INSTALL_DIR rev-parse --short HEAD)
+          fi
+
+          # Prepare historical test results directory
+          # a) Save text results to a directory with timestamp and commit hash
+          past_test_dir=$RPC_PAST_TEST_DIR/${CHAIN}_$(date +%Y%m%d_%H%M%S)_comp_perf_$commit
+          echo "past_test_dir=$past_test_dir" >> $GITHUB_ENV          
+          mkdir -p $past_test_dir
+          # b) Save binary results to a fixed directory
+          bin_past_test_dir=$RPC_PAST_TEST_DIR/${CHAIN}_bin
+          rm -rf $bin_past_test_dir  # we want only the latest binary files
+          mkdir -p $bin_past_test_dir
+
+          run_perf () {
+            workspace=$1
+            network=$2
+            method=$3
+            pattern=$4
+            sequence=$5
+            client=$6
+
+            result_dir=$workspace/rpc-tests/perf/reports/$network
+            result_file=$client-$method-result.json            
+
+            # clean temporary area
+            cd $workspace/rpc-tests/perf
+            rm -rf ./reports/$network
+
+            python3 ./run_perf_tests.py --blockchain "$network" \
+                                      --test-type "$method" \
+                                      --pattern-file pattern/"$network"/"$pattern".tar \
+                                      --test-sequence "$sequence" \
+                                      --repetitions 5 \
+                                      --erigon-dir "/" \
+                                      --silk-dir "/" \
+                                      --test-mode 2 \
+                                      --test-report \
+                                      --json-report $result_dir/$result_file \
+                                      --testing-daemon $client
+
+            # Capture test runner script exit status
+            perf_exit_status=$?
+
+            # Detect the pre-built db version
+            if [ "$client" == "erigon" ]; then
+              db_version=$(python3 $ERIGON_QA_PATH/test_system/qa-tests/uploads/prod_info.py $ERIGON_REFERENCE_DIR/production.ini production erigon_repo_commit)
+            else
+              db_version=$commit
+            fi
+
+            # Check test runner script exit status
+            if [ $perf_exit_status -eq 0 ]; then
+              outcome=success
+
+              # Save all vegeta binary reports
+              echo "Save current vegeta binary files"
+              cp -r $workspace/rpc-tests/perf/reports/bin $bin_past_test_dir
+
+              echo "Execute Latency Percentile HDR Analysis"
+              cd $result_dir
+              python3 $ERIGON_QA_PATH/test_system/qa-tests/rpc-tests/perf_hdr_analysis.py \
+                --test_name $client-$method \
+                --input_file ./$result_file \
+                --output_file ./$client-$method-latency_hdr_analysis.pdf
+            else
+              failed_test=1
+              outcome=failure
+            fi
+
+            # Save results on DB
+            echo "Save test result on DB"
+
+            if [ "$client" == "erigon" ]; then
+              branch_name=${{ github.ref_name }}
+              commit_hash=$(git -C $workspace/erigon rev-parse HEAD)
+            else
+              branch_name="release"
+              commit_hash=$commit
+            fi
+
+            echo branch_name=$branch_name
+            echo commit_hash=$commit_hash
+            echo method=$method
+            echo db_version=$db_version
+            echo outcome=$outcome
+            echo result_file=$result_file
+
+            python3 $ERIGON_QA_PATH/test_system/qa-tests/uploads/upload_test_results.py \
+              --repo $client \
+              --branch $branch_name \
+              --commit $commit_hash \
+              --test_name rpc-performance-comparison-test-$method \
+              --chain $CHAIN \
+              --runner ${{ runner.name }} \
+              --db_version $db_version \
+              --outcome $outcome \
+              --result_file $result_dir/$result_file
+
+            if [ $? -ne 0 ]; then
+              failed_test=1
+              echo "Failure saving test results on DB"
+            fi
+
+            # Save test results to a directory with timestamp and commit hash
+            cp -r $workspace/rpc-tests/perf/reports/$CHAIN $past_test_dir
+          }
+
+          # Launch the RPC performance test runner
+
+          #run_perf <workspace> <chain> <method> <pattern> <load-sequence> <client>
+          run_perf ${{runner.workspace}} $CHAIN eth_call stress_test_eth_call_001_14M 1:1,100:30,1000:20,10000:20,20000:20 $CLIENT                 
+
+          # Save the subsection reached status
+
+          echo "test_executed=true" >> $GITHUB_OUTPUT
+
+          if [ $failed_test -eq 0 ]; then
+            echo "TEST_RESULT=success" >> "$GITHUB_OUTPUT"
+            echo "Tests completed successfully"
+          else
+            echo "TEST_RESULT=failure" >> "$GITHUB_OUTPUT"
+            echo "Error detected during tests"
+            exit 1
+          fi
+
+      - name: Stop Erigon
+        if: always() && matrix.client == 'erigon'
+        working-directory: ${{ github.workspace }}/build/bin
+        run: |
+          # Clean up erigon process if it's still running
+          if [ -n "$ERIGON_PID" ] && kill -0 $ERIGON_PID 2> /dev/null; then
+            echo "Erigon stopping..."
+            kill $ERIGON_PID
+            echo "Erigon stopped"
+          else
+            echo "Erigon has already terminated"
+          fi
+
+      - name: Upload Erigon logs
+        if: matrix.client == 'erigon' && steps.test_step.outputs.test_executed == 'true'
+        uses: actions/upload-artifact@v6
+        with:
+          name: erigon-logs
+          path: ${{ env.ERIGON_TESTBED_DATA_DIR }}/logs/rpcdaemon.log
+
+      - name: Delete Erigon Testbed Data Directory
+        if: always() && matrix.client == 'erigon' && steps.save_datadir_step.outputs.datadir_saved == 'true'
+        run: |
+          rm -rf $ERIGON_TESTBED_DATA_DIR
+
+      - name: Resume the Erigon instance dedicated to db maintenance
+        if: always() && matrix.client == 'erigon'
+        run: |
+          python3 $ERIGON_QA_PATH/test_system/db-producer/resume_production.py || true