|
| 1 | +name: On-demand Benchmark |
| 2 | + |
| 3 | +on: |
| 4 | + pull_request_target: |
| 5 | + types: [labeled] |
| 6 | + |
| 7 | +concurrency: |
| 8 | + group: valkey-benchmark |
| 9 | + cancel-in-progress: false |
| 10 | + |
| 11 | +defaults: |
| 12 | + run: |
| 13 | + shell: "bash -Eeuo pipefail -x {0}" |
| 14 | + |
| 15 | +permissions: |
| 16 | + contents: read |
| 17 | + pull-requests: write |
| 18 | + issues: write |
| 19 | + |
| 20 | +jobs: |
| 21 | + benchmark: |
| 22 | + runs-on: ${{ github.repository == 'valkey-io/valkey' && fromJSON('["self-hosted","ec2-ubuntu-24.04-benchmarking"]') || 'ubuntu-latest' }} |
| 23 | + if: | |
| 24 | + github.event.action == 'labeled' && github.event.label.name == 'run-benchmark' |
| 25 | + steps: |
| 26 | + - name: Checkout valkey |
| 27 | + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 |
| 28 | + with: |
| 29 | + path: valkey |
| 30 | + fetch-depth: 0 |
| 31 | + ref: ${{ github.event.pull_request.merge_commit_sha }} |
| 32 | + persist-credentials: false |
| 33 | + |
| 34 | + - name: Checkout valkey-perf-benchmark |
| 35 | + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 |
| 36 | + with: |
| 37 | + repository: ${{ github.repository_owner }}/valkey-perf-benchmark |
| 38 | + path: valkey-perf-benchmark |
| 39 | + fetch-depth: 1 |
| 40 | + persist-credentials: false |
| 41 | + |
| 42 | + - name: Set up Python |
| 43 | + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 |
| 44 | + with: |
| 45 | + python-version: "3.10" |
| 46 | + cache: "pip" |
| 47 | + |
| 48 | + - name: Install dependencies |
| 49 | + working-directory: valkey-perf-benchmark |
| 50 | + run: | |
| 51 | + sudo apt-get update |
| 52 | + sudo apt-get install -y build-essential |
| 53 | + pip install -r requirements.txt |
| 54 | +
|
| 55 | + - name: Run benchmarks |
| 56 | + working-directory: valkey-perf-benchmark |
| 57 | + run: | |
| 58 | + CONFIG_FILE="../valkey/.github/benchmark_configs/pr_benchmark.json" |
| 59 | +
|
| 60 | + # Removes server/client CPU range args if running in private repo |
| 61 | + if [[ "${{ github.repository }}" != "valkey-io/valkey" ]]; then |
| 62 | + jq 'map(del(.server_cpu_range, .client_cpu_range))' "$CONFIG_FILE" > tmp.json && mv tmp.json "$CONFIG_FILE" |
| 63 | + fi |
| 64 | +
|
| 65 | + # Base benchmark arguments |
| 66 | + BENCHMARK_ARGS=( |
| 67 | + --commits "${{ github.event.pull_request.merge_commit_sha }}" |
| 68 | + --baseline "${{ github.event.pull_request.base.ref }}" |
| 69 | + --results-dir "results" |
| 70 | + --valkey-path "../valkey" |
| 71 | + --config "$CONFIG_FILE" |
| 72 | + ) |
| 73 | +
|
| 74 | + # Run benchmark |
| 75 | + python ./benchmark.py "${BENCHMARK_ARGS[@]}" |
| 76 | +
|
| 77 | + - name: Compare results |
| 78 | + working-directory: valkey-perf-benchmark |
| 79 | + run: | |
| 80 | + python ./utils/compare_benchmark_results.py \ |
| 81 | + ./results/${{ github.event.pull_request.base.ref }}/metrics.json \ |
| 82 | + ./results/${{ github.event.pull_request.merge_commit_sha }}/metrics.json \ |
| 83 | + ../comparison.md |
| 84 | +
|
| 85 | + - name: Upload artifacts |
| 86 | + if: always() |
| 87 | + continue-on-error: true |
| 88 | + uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0 |
| 89 | + with: |
| 90 | + name: benchmark-results |
| 91 | + path: | |
| 92 | + ./valkey-perf-benchmark/results/${{ github.event.pull_request.merge_commit_sha }}/metrics.json |
| 93 | + ./valkey-perf-benchmark/results/${{ github.event.pull_request.base.ref }}/metrics.json |
| 94 | + comparison.md |
| 95 | +
|
| 96 | + - name: Comment PR with results |
| 97 | + uses: actions/github-script@5c56fde4671bc2d3592fb0f2c5b5bab9ddae03b1 # v7 |
| 98 | + with: |
| 99 | + github-token: ${{ secrets.GITHUB_TOKEN }} |
| 100 | + script: | |
| 101 | + const fs = require('fs'); |
| 102 | + const body = fs.readFileSync('comparison.md', 'utf8'); |
| 103 | + const {owner, repo} = context.repo; |
| 104 | + const sha = '${{ github.event.pull_request.head.sha }}'; |
| 105 | + const short = sha.slice(0,7); |
| 106 | + const link = `[\`${short}\`](https://github.com/${owner}/${repo}/commit/${sha})` |
| 107 | + await github.rest.issues.createComment({ |
| 108 | + issue_number: context.issue.number, |
| 109 | + owner, |
| 110 | + repo, |
| 111 | + body: `**Benchmark ran on this commit:** ${link}\n\n${body}` |
| 112 | + }); |
| 113 | +
|
| 114 | + - name: Cleanup any running valkey processes |
| 115 | + if: always() |
| 116 | + continue-on-error: true |
| 117 | + run: | |
| 118 | + pkill -f valkey |
| 119 | + exit_code=$? |
| 120 | + if [ $exit_code -eq 0 ]; then |
| 121 | + echo "✓ Killed running valkey processes" |
| 122 | + elif [ $exit_code -eq 1 ]; then |
| 123 | + echo "No valkey processes found to kill" |
| 124 | + else |
| 125 | + echo "Warning: pkill failed with exit code $exit_code" |
| 126 | + fi |
| 127 | +
|
| 128 | + - name: Remove ${{ github.event.label.name }} label |
| 129 | + if: always() && github.event.label.name == 'run-benchmark' |
| 130 | + uses: actions/github-script@5c56fde4671bc2d3592fb0f2c5b5bab9ddae03b1 # v7 |
| 131 | + with: |
| 132 | + github-token: ${{ secrets.GITHUB_TOKEN }} |
| 133 | + script: | |
| 134 | + await github.rest.issues.removeLabel({ |
| 135 | + issue_number: context.issue.number, |
| 136 | + owner: context.repo.owner, |
| 137 | + repo: context.repo.repo, |
| 138 | + name: '${{ github.event.label.name }}' |
| 139 | + }).catch(err => console.log('label not present', err.message)); |
0 commit comments