Add support for manually running the real-time benchmark on PRs

.github/workflows/real-time-benchmark.yml

@@ -2,14 +2,81 @@ name: Real-time Benchmark
 on:
   schedule:
     - cron: "30 0 * * *"
+  workflow_dispatch:
+    inputs:
+      pull_request:
+        description: 'PR number that is going to be benchmarked (e.g. "1234")'
+        required: true
+        type: number
+      jit:
+        description: 'Whether JIT is benchmarked'
+        required: false
+        type: choice
+        options:
+          - "0"
+          - "1"
+      opcache:
+        description: 'Whether opcache is enabled for the benchmarked commit'
+        required: true
+        default: "1"
+        type: choice
+        options:
+          - "0"
+          - "1"
+          - "2"
+      baseline_opcache:
+        description: 'Whether opcache is enabled for the baseline commit'
+        required: true
+        default: "1"
+        type: choice
+        options:
+          - "0"
+          - "1"
+          - "2"
 permissions:
   contents: read
+  pull-requests: write
 jobs:
   REAL_TIME_BENCHMARK:
     name: REAL_TIME_BENCHMARK
-    if: github.repository == 'php/php-src'
+    if: github.repository == 'php/php-src' || github.event_name == 'workflow_dispatch'
     runs-on: ubuntu-22.04
+    env:
+      REPOSITORY: ${{ github.repository }}
+      BRANCH: "master"
+      COMMIT: ${{ github.sha }}
+      BASELINE_COMMIT: "d5f6e56610c729710073350af318c4ea1b292cfe"
+      ID: "master"
+      OPCACHE: ${{ inputs.opcache || '1' }}
+      BASELINE_OPCACHE: ${{ inputs.baseline_opcache || '2' }}
+      JIT: ${{ inputs.jit || '1' }}
+      YEAR: ""
     steps:
+      - name: Setup benchmark environment
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          YEAR="$(date '+%Y')"
+          echo "YEAR=$YEAR" >> $GITHUB_ENV
+
+          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+            PR_INFO=$(gh pr view ${{ inputs.pull_request }} --json headRepositoryOwner,headRepository,headRefName,headRefOid,baseRefOid --repo ${{ github.repository }} | jq -r '.headRepositoryOwner.login, .headRepository.name, .headRefName, .headRefOid, .baseRefOid')
+
+            REPOSITORY="$(echo "$PR_INFO" | sed -n '1p')/$(echo "$PR_INFO" | sed -n '2p')"
+            echo "REPOSITORY=$REPOSITORY" >> $GITHUB_ENV
+
+            BRANCH=$(echo "$PR_INFO" | sed -n '3p')
+            echo "BRANCH=$BRANCH" >> $GITHUB_ENV
+
+            COMMIT=$(echo "$PR_INFO" | sed -n '4p')
+            echo "COMMIT=$COMMIT" >> $GITHUB_ENV
+
+            BASELINE_COMMIT=$(echo "$PR_INFO" | sed -n '5p')
+            echo "BASELINE_COMMIT=$BASELINE_COMMIT" >> $GITHUB_ENV
+
+            echo "ID=benchmarked" >> $GITHUB_ENV
+          fi
+
       - name: Install dependencies
         run: |
           set -ex
@@ -29,73 +96,142 @@ jobs:
           ref: 'main'
           fetch-depth: 1
           path: 'php-version-benchmarks'
-      - name: Checkout php-src
+      - name: Checkout php-src (benchmarked version)
+        uses: actions/checkout@v4
+        with:
+          repository: '${{ env.REPOSITORY }}'
+          ref: '${{ env.COMMIT }}'
+          fetch-depth: 100
+          path: 'php-version-benchmarks/tmp/php_${{ env.ID }}'
+      - name: Checkout php-src (baseline version)
         uses: actions/checkout@v4
         with:
-          repository: 'php/php-src'
-          ref: '${{ github.sha }}'
+          repository: '${{ env.REPOSITORY }}'
+          ref: '${{ env.BASELINE_COMMIT }}'
           fetch-depth: 100
-          path: 'php-version-benchmarks/tmp/php_master'
+          path: 'php-version-benchmarks/tmp/php_baseline'
       - name: Setup benchmark results
         run: |
           git config --global user.name "Benchmark"
           git config --global user.email "[email protected]"
-          
+
           rm -rf ./php-version-benchmarks/docs/results
       - name: Checkout benchmark data
+        if: github.event_name != 'workflow_dispatch'
         uses: actions/checkout@v4
         with:
           repository: php/real-time-benchmark-data
           ssh-key: ${{ secrets.PHP_VERSION_BENCHMARK_RESULTS_DEPLOY_KEY }}
           path: 'php-version-benchmarks/docs/results'
-      - name: Set benchmark config
+      - name: Setup infra config
         run: |
           set -e
 
-          # Set infrastructure config
           cp ./php-version-benchmarks/config/infra/aws/x86_64-metal.ini.dist ./php-version-benchmarks/config/infra/aws/x86_64-metal.ini
           ESCAPED_DOCKER_REGISTRY=$(printf '%s\n' "${{ secrets.PHP_VERSION_BENCHMARK_DOCKER_REGISTRY }}" | sed -e 's/[\/&]/\\&/g')
           sed -i "s/INFRA_DOCKER_REGISTRY=public.ecr.aws\/abcdefgh/INFRA_DOCKER_REGISTRY=$ESCAPED_DOCKER_REGISTRY/g" ./php-version-benchmarks/config/infra/aws/x86_64-metal.ini
           cp ./php-version-benchmarks/build/infrastructure/config/aws.tfvars.dist ./php-version-benchmarks/build/infrastructure/config/aws.tfvars
           sed -i 's/access_key = ""/access_key = "${{ secrets.PHP_VERSION_BENCHMARK_AWS_ACCESS_KEY }}"/g' ./php-version-benchmarks/build/infrastructure/config/aws.tfvars
           sed -i 's/secret_key = ""/secret_key = "${{ secrets.PHP_VERSION_BENCHMARK_AWS_SECRET_KEY }}"/g' ./php-version-benchmarks/build/infrastructure/config/aws.tfvars
+          sed -i 's/github_token = ""/github_token = "${{ secrets.GITHUB_TOKEN }}"/g' ./php-version-benchmarks/build/infrastructure/config/aws.tfvars
+      - name: Setup PHP config - baseline PHP version
+        run: |
+          set -e
 
-          YEAR="$(date '+%Y')"
-          DATABASE="./php-version-benchmarks/docs/results/$YEAR/database.tsv"
+          BASELINE_SHORT_SHA="$(echo "${{ env.BASELINE_COMMIT }}" | cut -c1-4)"
+
+          cat << EOF > ./php-version-benchmarks/config/php/baseline.ini
+            PHP_NAME="PHP - baseline@$BASELINE_SHORT_SHA"
+            PHP_ID=php_baseline
+
+            PHP_REPO=https://github.com/${{ env.REPOSITORY }}.git
+            PHP_BRANCH=${{ env.BRANCH }}
+            PHP_COMMIT=${{ env.BASELINE_COMMIT }}
+
+            PHP_OPCACHE=${{ env.BASELINE_OPCACHE }}
+            PHP_JIT=0
+          EOF
+      - name: Setup PHP config - baseline PHP version with JIT
+        if: github.event_name == 'workflow_dispatch' && inputs.jit == '1'
+        run: |
+          set -e
+
+          BASELINE_SHORT_SHA="$(echo "${{ env.BASELINE_COMMIT }}" | cut -c1-4)"
+
+          cat << EOF > ./php-version-benchmarks/config/php/baseline_jit.ini
+            PHP_NAME="PHP - baseline@$BASELINE_SHORT_SHA (JIT)"
+            PHP_ID=php_baseline_jit
+
+            PHP_REPO=https://github.com/${{ env.REPOSITORY }}.git
+            PHP_BRANCH=${{ env.BRANCH }}
+            PHP_COMMIT=${{ env.BASELINE_COMMIT }}
+
+            PHP_OPCACHE=${{ env.BASELINE_OPCACHE }}
+            PHP_JIT=${{ env.JIT }}
+          EOF
+
+          git clone ./php-version-benchmarks/tmp/php_baseline/ ./php-version-benchmarks/tmp/php_baseline_jit
+      - name: Setup PHP config - previous PHP version
+        if: github.event_name != 'workflow_dispatch'
+        run: |
+          set -e
+
+          DATABASE="./php-version-benchmarks/docs/results/${{ env.YEAR }}/database.tsv"
           if [ -f "$DATABASE" ]; then
             LAST_RESULT_SHA="$(tail -n 2 "$DATABASE" | head -n 1 | cut -f 6)"
           else
             YESTERDAY="$(date -d "-2 day 23:59:59" '+%Y-%m-%d %H:%M:%S')"
-            LAST_RESULT_SHA="$(cd ./php-version-benchmarks/tmp/php_master/ && git --no-pager log --until="$YESTERDAY" -n 1 --pretty='%H')"
+            LAST_RESULT_SHA="$(cd ./php-version-benchmarks/tmp/php_${{ env.ID }}/ && git --no-pager log --until="$YESTERDAY" -n 1 --pretty='%H')"
           fi
 
-          BASELINE_SHA="d5f6e56610c729710073350af318c4ea1b292cfe"
-          BASELINE_SHORT_SHA="$(echo "$BASELINE_SHA" | cut -c1-4)"
-
-          # Set config for the baseline PHP version
-          cp ./php-version-benchmarks/config/php/master.ini.dist ./php-version-benchmarks/config/php/master_baseline.ini
-          sed -i 's/PHP_NAME="PHP - master"/PHP_NAME="PHP - baseline@'"$BASELINE_SHORT_SHA"'"/g' ./php-version-benchmarks/config/php/master_baseline.ini
-          sed -i "s/PHP_ID=php_master/PHP_ID=php_master_baseline/g" ./php-version-benchmarks/config/php/master_baseline.ini
-          sed -i "s/PHP_COMMIT=/PHP_COMMIT=$BASELINE_SHA/g" ./php-version-benchmarks/config/php/master_baseline.ini
-          sed -i "s/PHP_OPCACHE=1/PHP_OPCACHE=2/g" ./php-version-benchmarks/config/php/master_baseline.ini
-
-          # Set config for the previous PHP version
-          cp ./php-version-benchmarks/config/php/master.ini.dist ./php-version-benchmarks/config/php/master_last.ini
-          sed -i 's/PHP_NAME="PHP - master"/PHP_NAME="PHP - previous master"/g' ./php-version-benchmarks/config/php/master_last.ini
-          sed -i "s/PHP_ID=php_master/PHP_ID=php_master_previous/g" ./php-version-benchmarks/config/php/master_last.ini
-          sed -i "s/PHP_COMMIT=/PHP_COMMIT=$LAST_RESULT_SHA/g" ./php-version-benchmarks/config/php/master_last.ini
-          sed -i "s/PHP_OPCACHE=1/PHP_OPCACHE=2/g" ./php-version-benchmarks/config/php/master_last.ini
-
-          # Set config for the current PHP version
-          cp ./php-version-benchmarks/config/php/master.ini.dist ./php-version-benchmarks/config/php/master_now.ini
-          sed -i "s/PHP_COMMIT=/PHP_COMMIT=${{ github.sha }}/g" ./php-version-benchmarks/config/php/master_now.ini
-
-          # Set config for current PHP version with JIT
-          git clone ./php-version-benchmarks/tmp/php_master/ ./php-version-benchmarks/tmp/php_master_jit
-          cp ./php-version-benchmarks/config/php/master_jit.ini.dist ./php-version-benchmarks/config/php/master_now_jit.ini
-          sed -i "s/PHP_COMMIT=/PHP_COMMIT=${{ github.sha }}/g" ./php-version-benchmarks/config/php/master_now_jit.ini
-
-          # Set test configs
+          cat << EOF > ./php-version-benchmarks/config/php/previous.ini
+            PHP_NAME="PHP - previous ${{ env.BRANCH }}"
+            PHP_ID=php_previous
+
+            PHP_REPO=https://github.com/${{ env.REPOSITORY }}.git
+            PHP_BRANCH=${{ env.BRANCH }}
+            PHP_COMMIT=$LAST_RESULT_SHA
+
+            PHP_OPCACHE=1
+            PHP_JIT=0
+          EOF
+      - name: Setup PHP config - benchmarked PHP version
+        run: |
+          set -e
+
+          cat << EOF > ./php-version-benchmarks/config/php/this.ini
+            PHP_NAME="PHP - ${{ env.BRANCH }}"
+            PHP_ID=php_${{ env.ID }}
+
+            PHP_REPO=https://github.com/${{ env.REPOSITORY }}.git
+            PHP_BRANCH=${{ env.BRANCH }}
+            PHP_COMMIT=${{ env.COMMIT }}
+
+            PHP_OPCACHE=${{ env.OPCACHE }}
+            PHP_JIT=0
+          EOF
+      - name: Setup PHP config - benchmarked PHP version with JIT
+        if: env.JIT == '1'
+        run: |
+          set -e
+
+          cat << EOF > ./php-version-benchmarks/config/php/this_jit.ini
+            PHP_NAME="PHP - ${{ env.BRANCH }} (JIT)"
+            PHP_ID=php_${{ env.ID }}_jit
+
+            PHP_REPO=https://github.com/${{ env.REPOSITORY }}.git
+            PHP_BRANCH=${{ env.BRANCH }}
+            PHP_COMMIT=${{ env.COMMIT }}
+
+            PHP_OPCACHE=${{ env.OPCACHE }}
+            PHP_JIT=${{ env.JIT }}
+          EOF
+
+          git clone ./php-version-benchmarks/tmp/php_${{ env.ID }}/ ./php-version-benchmarks/tmp/php_${{ env.ID }}_jit
+      - name: Setup test config
+        run: |
+          set -e
+
           cp ./php-version-benchmarks/config/test/1_laravel.ini.dist ./php-version-benchmarks/config/test/1_laravel.ini
           cp ./php-version-benchmarks/config/test/2_symfony_main.ini.dist ./php-version-benchmarks/config/test/2_symfony_main.ini
           cp ./php-version-benchmarks/config/test/4_wordpress.ini.dist ./php-version-benchmarks/config/test/4_wordpress.ini
@@ -104,6 +240,7 @@ jobs:
       - name: Run benchmark
         run: ./php-version-benchmarks/benchmark.sh run aws
       - name: Store results
+        if: github.repository == 'php/php-src' && github.event_name != 'workflow_dispatch'
         run: |
           set -ex
 
@@ -119,6 +256,21 @@ jobs:
           fi
           git commit -m "Add result for ${{ github.repository }}@${{ github.sha }}"
           git push
+      - name: Upload artifact
+        if: github.event_name == 'workflow_dispatch'
+        uses: actions/upload-artifact@v4
+        with:
+          name: results
+          path: ./php-version-benchmarks/docs/results/${{ env.YEAR }}
+          retention-days: 30
+      - name: Comment results
+        if: github.event_name == 'workflow_dispatch'
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          cd ./php-version-benchmarks/tmp/php_${{ env.ID }}
+          NEWEST_RESULT_DIRECTORY=$(ls -td ${{ github.workspace }}/php-version-benchmarks/docs/results/${{ env.YEAR }}/*/ | head -1)
+          gh pr comment ${{ inputs.pull_request }} --body-file "${NEWEST_RESULT_DIRECTORY}result.md" --repo ${{ env.REPOSITORY }}
       - name: Cleanup
         if: always()
         run: |