diff --git a/.github/workflows/qa-rpc-performance-comparison-tests.yml b/.github/workflows/qa-rpc-performance-comparison-tests.yml index 4d4451fa9f2..aa269db3512 100644 --- a/.github/workflows/qa-rpc-performance-comparison-tests.yml +++ b/.github/workflows/qa-rpc-performance-comparison-tests.yml @@ -11,11 +11,12 @@ concurrency: env: CHAIN: mainnet ERIGON_QA_PATH: /home/qarunner/erigon-qa + RPC_PAST_TEST_DIR: /opt/rpc-past-tests jobs: setup: name: Setup benchmark run - runs-on: ubuntu-latest + runs-on: [self-hosted, qa] permissions: contents: read outputs: @@ -61,17 +62,25 @@ jobs: - name: Prepare Redis database shell: bash - working-directory: ${{ env.ERIGON_QA_PATH }}/test_system/qa-tests/tip-tracking/rpc-tests + working-directory: ${{ env.ERIGON_QA_PATH }}/test_system/qa-tests/rpc-tests run: | set -euo pipefail - python redis_client.py --key ${{ steps.vars.outputs.erigon_redis_key }} --write "starting" - python redis_client.py --key ${{ steps.vars.outputs.geth_redis_key }} --write "starting" - - test_erigon: - name: Benchmark Erigon - needs: [ setup ] - runs-on: [ self-hosted, qa, Ethereum, rpc-latest-erigon ] - timeout-minutes: 120 + python3 redis_client.py --key ${{ steps.vars.outputs.erigon_redis_key }} --write "created" + python3 redis_client.py --key ${{ steps.vars.outputs.geth_redis_key }} --write "created" + + test: + name: Benchmark ${{ matrix.client }} + needs: [setup] + strategy: + fail-fast: true + matrix: + include: + - client: erigon + runs-on: [self-hosted, qa, Ethereum, rpc-latest-erigon] + - client: geth + runs-on: [self-hosted, qa, Ethereum, rpc-latest-geth] + runs-on: ${{ matrix.runs-on }} + timeout-minutes: 480 permissions: contents: read env: @@ -81,62 +90,110 @@ jobs: VEGETA_DURATION: ${{ needs.setup.outputs.vegeta_duration }} ERIGON_REDIS_KEY: ${{ needs.setup.outputs.erigon_redis_key }} GETH_REDIS_KEY: ${{ needs.setup.outputs.geth_redis_key }} + ERIGON_REFERENCE_DIR: /opt/erigon-versions/reference-version + ERIGON_REFERENCE_DATA_DIR: /opt/erigon-versions/reference-version/datadir + ERIGON_TESTBED_DATA_DIR: /opt/erigon-testbed/datadir + GETH_INSTALL_DIR: /opt/go-ethereum + CLIENT: ${{ matrix.client }} steps: - name: Checkout uses: actions/checkout@v6 - name: Sanity check Redis config shell: bash - working-directory: ${{ env.ERIGON_QA_PATH }}/test_system/qa-tests/tip-tracking/rpc-tests + working-directory: ${{ env.ERIGON_QA_PATH }}/test_system/qa-tests/rpc-tests run: | set -euo pipefail - python redis_client.py --key ${{ needs.setup.outputs.erigon_redis_key }} --read + python3 redis_client.py --key ${{ matrix.client == 'erigon' && needs.setup.outputs.erigon_redis_key || needs.setup.outputs.geth_redis_key }} --read - - name: Run Erigon benchmark (placeholder) - shell: bash + - name: Checkout RPC Tests Repository run: | - set -euo pipefail - echo "TODO: Build/run Erigon, monitor sync, publish status to Redis, wait barrier, run Vegeta." - echo "Context:" - echo " RUN_ID=${RUN_ID}" - echo " REDIS_KEY_PREFIX=${REDIS_KEY_PREFIX}" - echo " NETWORK=${NETWORK}" - echo " VEGETA_DURATION=${VEGETA_DURATION}" - echo " ERIGON_REDIS_KEY=${ERIGON_REDIS_KEY}" - echo " GETH_REDIS_KEY=${GETH_REDIS_KEY}" - # Example placeholders for later: - # python3 scripts/bench_client.py --client erigon --redis-prefix "REDIS_KEY_PREFIX" --network "$NETWORK" - - test_geth: - name: Benchmark Geth - needs: [ setup ] - runs-on: [ self-hosted, qa, Ethereum, rpc-latest-geth ] - timeout-minutes: 120 - permissions: - contents: read - env: - RUN_ID: ${{ needs.setup.outputs.run_id }} - REDIS_KEY_PREFIX: ${{ needs.setup.outputs.redis_key_prefix }} - NETWORK: ${{ needs.setup.outputs.network }} - VEGETA_DURATION: ${{ needs.setup.outputs.vegeta_duration }} - ERIGON_REDIS_KEY: ${{ needs.setup.outputs.erigon_redis_key }} - GETH_REDIS_KEY: ${{ needs.setup.outputs.geth_redis_key }} - steps: - - name: Checkout - uses: actions/checkout@v6 + rm -rf ${{runner.workspace}}/rpc-tests + git -c advice.detachedHead=false clone --depth 1 --branch v1.78.0 https://github.com/erigontech/rpc-tests ${{runner.workspace}}/rpc-tests + cd ${{runner.workspace}}/rpc-tests - - name: Sanity check Redis config - shell: bash - working-directory: ${{ env.ERIGON_QA_PATH }}/test_system/qa-tests/tip-tracking/rpc-tests + - name: Clean Erigon Build Directory + if: matrix.client == 'erigon' run: | - set -euo pipefail - python redis_client.py --key ${{ needs.setup.outputs.geth_redis_key }} --read + make clean + + - name: Build Erigon + if: matrix.client == 'erigon' + run: | + make erigon integration + working-directory: ${{ github.workspace }} + + - name: Pause the Erigon instance dedicated to db maintenance + if: matrix.client == 'erigon' + run: | + python3 $ERIGON_QA_PATH/test_system/db-producer/pause_production.py || true + + - name: Save Erigon datadir Directory + if: matrix.client == 'erigon' + id: save_datadir_step + run: | + rm -rf $ERIGON_TESTBED_DATA_DIR || true + + echo "Mirror datadir" + if ! "$GITHUB_WORKSPACE/cmd/scripts/mirror-datadir.sh" "$ERIGON_REFERENCE_DATA_DIR" "$ERIGON_TESTBED_DATA_DIR" > /dev/null 2>&1; then + echo "Failed to mirror datadir from $ERIGON_REFERENCE_DATA_DIR to $ERIGON_TESTBED_DATA_DIR" + exit 1 + fi + echo "datadir_saved=true" >> $GITHUB_OUTPUT + + - name: Run Migrations + if: matrix.client == 'erigon' + working-directory: ${{ github.workspace }}/build/bin + run: | + echo "Running migrations on datadir..." + ./integration run_migrations --datadir $ERIGON_TESTBED_DATA_DIR --chain $CHAIN - - name: Run Geth benchmark (placeholder) + - name: Run Erigon + if: matrix.client == 'erigon' + id: erigon_running_step + working-directory: ${{ github.workspace }}/build/bin + run: | + set +e # Disable exit on error + echo "Starting Erigon..." + + ./erigon --prune.mode=minimal --datadir $ERIGON_TESTBED_DATA_DIR --http.api admin,debug,eth,parity,erigon,trace,web3,txpool,ots,net --ws > erigon.log 2>&1 & + + ERIGON_PID=$! + echo "ERIGON_PID=$ERIGON_PID" >> $GITHUB_ENV + echo "erigon_started=true" >> $GITHUB_OUTPUT + + sleep 5 + tail erigon.log + + if ! kill -0 "$ERIGON_PID" 2>/dev/null; then + echo "Erigon failed to start" + echo "::error::Error detected during tests: Erigon failed to start" + exit 1 + fi + echo "Erigon started" + + - name: Wait for port 8545 to be opened + run: | + for i in {1..30}; do + if nc -z localhost 8545; then + echo "Port 8545 is open" + break + fi + echo "Waiting for port 8545 to open..." + sleep 10 + done + if ! nc -z localhost 8545; then + echo "Port 8545 did not open in time" + echo "::error::Error detected during tests: Port 8545 did not open in time" + exit 1 + fi + + - name: Find a consensus on a common time to run the benchmark shell: bash + working-directory: ${{ env.ERIGON_QA_PATH }}/test_system/qa-tests/rpc-tests run: | set -euo pipefail - echo "TODO: Monitor Geth sync, publish status to Redis, wait barrier, run Vegeta." + echo "Monitor sync, publish status to Redis, wait barrier, run Vegeta." echo "Context:" echo " RUN_ID=${RUN_ID}" echo " REDIS_KEY_PREFIX=${REDIS_KEY_PREFIX}" @@ -144,5 +201,170 @@ jobs: echo " VEGETA_DURATION=${VEGETA_DURATION}" echo " ERIGON_REDIS_KEY=${ERIGON_REDIS_KEY}" echo " GETH_REDIS_KEY=${GETH_REDIS_KEY}" - # Example placeholders for later: - # python3 scripts/bench_client.py --client geth --redis-prefix "REDIS_KEY_PREFIX" --network "$NETWORK" \ No newline at end of file + # Wait for an agreement between Erigon and Geth + python3 coordinated_start.py --client "$CLIENT" --run-id "$RUN_ID" --chain "$NETWORK" + + - name: Run RPC Performance Tests + id: test_step + run: | + set +e # Disable exit on error + + failed_test=0 + + if [ "${{ matrix.client }}" == "erigon" ]; then + commit=$(git -C ${{runner.workspace}}/erigon rev-parse --short HEAD) + else + commit=$(git -C $GETH_INSTALL_DIR rev-parse --short HEAD) + fi + + # Prepare historical test results directory + # a) Save text results to a directory with timestamp and commit hash + past_test_dir=$RPC_PAST_TEST_DIR/${CHAIN}_$(date +%Y%m%d_%H%M%S)_comp_perf_$commit + echo "past_test_dir=$past_test_dir" >> $GITHUB_ENV + mkdir -p $past_test_dir + # b) Save binary results to a fixed directory + bin_past_test_dir=$RPC_PAST_TEST_DIR/${CHAIN}_bin + rm -rf $bin_past_test_dir # we want only the latest binary files + mkdir -p $bin_past_test_dir + + run_perf () { + workspace=$1 + network=$2 + method=$3 + pattern=$4 + sequence=$5 + client=$6 + + result_dir=$workspace/rpc-tests/perf/reports/$network + result_file=$client-$method-result.json + + # clean temporary area + cd $workspace/rpc-tests/perf + rm -rf ./reports/$network + + python3 ./run_perf_tests.py --blockchain "$network" \ + --test-type "$method" \ + --pattern-file pattern/"$network"/"$pattern".tar \ + --test-sequence "$sequence" \ + --repetitions 5 \ + --erigon-dir "/" \ + --silk-dir "/" \ + --test-mode 2 \ + --test-report \ + --json-report $result_dir/$result_file \ + --testing-daemon $client + + # Capture test runner script exit status + perf_exit_status=$? + + # Detect the pre-built db version + if [ "$client" == "erigon" ]; then + db_version=$(python3 $ERIGON_QA_PATH/test_system/qa-tests/uploads/prod_info.py $ERIGON_REFERENCE_DIR/production.ini production erigon_repo_commit) + else + db_version=$commit + fi + + # Check test runner script exit status + if [ $perf_exit_status -eq 0 ]; then + outcome=success + + # Save all vegeta binary reports + echo "Save current vegeta binary files" + cp -r $workspace/rpc-tests/perf/reports/bin $bin_past_test_dir + + echo "Execute Latency Percentile HDR Analysis" + cd $result_dir + python3 $ERIGON_QA_PATH/test_system/qa-tests/rpc-tests/perf_hdr_analysis.py \ + --test_name $client-$method \ + --input_file ./$result_file \ + --output_file ./$client-$method-latency_hdr_analysis.pdf + else + failed_test=1 + outcome=failure + fi + + # Save results on DB + echo "Save test result on DB" + + if [ "$client" == "erigon" ]; then + branch_name=${{ github.ref_name }} + commit_hash=$(git -C $workspace/erigon rev-parse HEAD) + else + branch_name="release" + commit_hash=$commit + fi + + echo branch_name=$branch_name + echo commit_hash=$commit_hash + echo method=$method + echo db_version=$db_version + echo outcome=$outcome + echo result_file=$result_file + + python3 $ERIGON_QA_PATH/test_system/qa-tests/uploads/upload_test_results.py \ + --repo $client \ + --branch $branch_name \ + --commit $commit_hash \ + --test_name rpc-performance-comparison-test-$method \ + --chain $CHAIN \ + --runner ${{ runner.name }} \ + --db_version $db_version \ + --outcome $outcome \ + --result_file $result_dir/$result_file + + if [ $? -ne 0 ]; then + failed_test=1 + echo "Failure saving test results on DB" + fi + + # Save test results to a directory with timestamp and commit hash + cp -r $workspace/rpc-tests/perf/reports/$CHAIN $past_test_dir + } + + # Launch the RPC performance test runner + + #run_perf + run_perf ${{runner.workspace}} $CHAIN eth_call stress_test_eth_call_001_14M 1:1,100:30,1000:20,10000:20,20000:20 $CLIENT + + # Save the subsection reached status + + echo "test_executed=true" >> $GITHUB_OUTPUT + + if [ $failed_test -eq 0 ]; then + echo "TEST_RESULT=success" >> "$GITHUB_OUTPUT" + echo "Tests completed successfully" + else + echo "TEST_RESULT=failure" >> "$GITHUB_OUTPUT" + echo "Error detected during tests" + exit 1 + fi + + - name: Stop Erigon + if: always() && matrix.client == 'erigon' + working-directory: ${{ github.workspace }}/build/bin + run: | + # Clean up erigon process if it's still running + if [ -n "$ERIGON_PID" ] && kill -0 $ERIGON_PID 2> /dev/null; then + echo "Erigon stopping..." + kill $ERIGON_PID + echo "Erigon stopped" + else + echo "Erigon has already terminated" + fi + + - name: Upload Erigon logs + if: matrix.client == 'erigon' && steps.test_step.outputs.test_executed == 'true' + uses: actions/upload-artifact@v6 + with: + name: erigon-logs + path: ${{ env.ERIGON_TESTBED_DATA_DIR }}/logs/rpcdaemon.log + + - name: Delete Erigon Testbed Data Directory + if: always() && matrix.client == 'erigon' && steps.save_datadir_step.outputs.datadir_saved == 'true' + run: | + rm -rf $ERIGON_TESTBED_DATA_DIR + + - name: Resume the Erigon instance dedicated to db maintenance + if: always() && matrix.client == 'erigon' + run: | + python3 $ERIGON_QA_PATH/test_system/db-producer/resume_production.py || true