Merge branch 'main' into dev/kpietkun/tests_custom_op_correctness

Author: Kacper-Pietkun

.cd/Dockerfile.rhel.tenc.pytorch.vllm

@@ -61,7 +61,7 @@ RUN \
     git fetch upstream --tags || true && \
     git checkout ${VLLM_PROJECT_COMMIT} && \
     # Install vllm-project/vllm
-    bash -c "pip install -r <(sed '/^[torch]/d' requirements/build.txt)" && \
+    bash -c "pip install -r <(sed '/^torch/d' requirements/build.txt)" && \
     VLLM_TARGET_DEVICE=empty pip install --no-build-isolation . && \
     # Install vllm-gaudi plugin
     cd $VLLM_PATH2 && \

.cd/Dockerfile.ubuntu.pytorch.vllm

@@ -51,7 +51,7 @@ RUN \
     git fetch upstream --tags || true && \
     git checkout ${VLLM_PROJECT_COMMIT} && \
     # Install vllm-project/vllm
-    bash -c "pip install -r <(sed '/^[torch]/d' requirements/build.txt)" && \
+    bash -c "pip install -r <(sed '/^torch/d' requirements/build.txt)" && \
     VLLM_TARGET_DEVICE=empty pip install --no-build-isolation . && \
     # Install vllm-gaudi plugin
     cd $VLLM_PATH2 && \

.cd/Dockerfile.ubuntu.pytorch.vllm.nixl.latest

@@ -45,7 +45,7 @@ RUN \
     git remote add upstream https://github.com/vllm-project/vllm.git && \
     git fetch upstream --tags || true && \
     git checkout ${VLLM_COMMIT_HASH} && \
-    pip install -r <(sed '/^[torch]/d' requirements/build.txt) && \
+    pip install -r <(sed '/^torch/d' requirements/build.txt) && \
     VLLM_TARGET_DEVICE=empty pip install --no-build-isolation . && \
     \
     # Install vllm-gaudi

.github/actionlint.yaml

@@ -2,3 +2,4 @@ self-hosted-runner:
   labels:
     - ucb-vllm-cicd-g2
     - hourly-ci
+    - pr-ci

.github/workflows/create-release-branch.yaml

@@ -120,9 +120,27 @@ jobs:
             echo "branch_name=pre_${{ github.event.inputs.branch_name }}" >> "$GITHUB_OUTPUT"
           fi
 
-  setup_and_build:
-    runs-on: hourly-ci
+  # --- NEW JOB ---
+  # This job runs after prep, picks one 'hourly-ci' runner,
+  # and outputs its name so all other test jobs can target it.
+  discover_runner:
+    name: "Discover Self-Hosted Runner"
     needs: [prepare-release-branch]
+    runs-on: hourly-ci
+    outputs:
+      runner_name: ${{ steps.get_name.outputs.name }}
+    steps:
+      - name: Get runner name
+        id: get_name
+        run: |
+          echo "This workflow will run on: ${{ runner.name }}"
+          echo "name=${{ runner.name }}" >> "$GITHUB_OUTPUT"
+
+  setup_and_build:
+    # --- UPDATED: Add discover_runner dependency ---
+    needs: [prepare-release-branch, discover_runner]
+    # --- UPDATED: Run on the specific node ---
+    runs-on: ${{ needs.discover_runner.outputs.runner_name }}
     steps:
       - name: "Checkout the release branch"
         uses: actions/checkout@v4
@@ -168,8 +186,10 @@ jobs:
           echo "Docker image built successfully."
 
   run_unit_tests:
-    needs: [prepare-release-branch, setup_and_build]
-    runs-on: hourly-ci
+    # --- UPDATED: Add discover_runner dependency ---
+    needs: [prepare-release-branch, setup_and_build, discover_runner]
+    # --- UPDATED: Run on the specific node ---
+    runs-on: ${{ needs.discover_runner.outputs.runner_name }}
     steps:
       - name: Run pytest in tests/unit_tests
         run: |
@@ -191,8 +211,10 @@ jobs:
           echo "Test script exited with code: $EXITCODE"
 
   discover_tests:
-    runs-on: hourly-ci
-    needs: [prepare-release-branch]
+    # --- UPDATED: Add discover_runner dependency ---
+    needs: [prepare-release-branch, discover_runner]
+    # --- UPDATED: Run on the specific node ---
+    runs-on: ${{ needs.discover_runner.outputs.runner_name }}
     outputs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
     steps:
@@ -204,9 +226,9 @@ jobs:
         id: set-matrix
         run: |
           TEST_FUNCTIONS=$( grep '^run_' ./tests/full_tests/ci_gsm8k_tests.sh | \
-              awk '{print $1}' | \
-              sed 's/()//' | \
-              jq -R . | jq -s -c . )
+                      awk '{print $1}' | \
+                      sed 's/()//' | \
+                      jq -R . | jq -s -c . )
 
           echo "Discovered test matrix: $TEST_FUNCTIONS"
           if [ "$TEST_FUNCTIONS" = "[]" ]; then
@@ -216,8 +238,10 @@ jobs:
           echo "matrix=$TEST_FUNCTIONS" >> "$GITHUB_OUTPUT"
 
   e2e:
-    needs: [prepare-release-branch, setup_and_build, discover_tests]
-    runs-on: hourly-ci
+    # --- UPDATED: Add discover_runner dependency ---
+    needs: [prepare-release-branch, setup_and_build, discover_tests, discover_runner]
+    # --- UPDATED: Run on the specific node ---
+    runs-on: ${{ needs.discover_runner.outputs.runner_name }}
     strategy:
       fail-fast: false
       matrix:
@@ -245,8 +269,10 @@ jobs:
           echo "Test script exited with code: $EXITCODE"
 
   run_data_parallel_test:
-    needs: [prepare-release-branch, setup_and_build]
-    runs-on: hourly-ci
+    # --- UPDATED: Add discover_runner dependency ---
+    needs: [prepare-release-branch, setup_and_build, discover_runner]
+    # --- UPDATED: Run on the specific node ---
+    runs-on: ${{ needs.discover_runner.outputs.runner_name }}
     steps:
       - name: Run Data Parallel test
         run: |
@@ -271,8 +297,10 @@ jobs:
           echo "Test script exited with code: $EXITCODE"
 
   run_pd_disaggregate_test:
-    needs: [prepare-release-branch, setup_and_build]
-    runs-on: hourly-ci
+    # --- UPDATED: Add discover_runner dependency ---
+    needs: [prepare-release-branch, setup_and_build, discover_runner]
+    # --- UPDATED: Run on the specific node ---
+    runs-on: ${{ needs.discover_runner.outputs.runner_name }}
     steps:
       - name: Run PD disaggregate test
         run: |
@@ -298,8 +326,10 @@ jobs:
           echo "Test script exited with code: $EXITCODE"
 
   run_hpu_perf_tests:
-    needs: [prepare-release-branch, setup_and_build]
-    runs-on: hourly-ci
+    # --- UPDATED: Add discover_runner dependency ---
+    needs: [prepare-release-branch, setup_and_build, discover_runner]
+    # --- UPDATED: Run on the specific node ---
+    runs-on: ${{ needs.discover_runner.outputs.runner_name }}
     steps:
       - name: Run Sharegpt performance tests with warmup
         run: |
@@ -324,6 +354,8 @@ jobs:
   summarize_and_notify:
     name: "Summarize Test Results and Notify"
     runs-on: ubuntu-latest
+    # --- This job runs on ubuntu-latest, so no runner change is needed ---
+    # It will correctly wait for all the test jobs to finish
     if: needs.prepare-release-branch.result == 'success'
     needs:
       - prepare-release-branch

.github/workflows/hourly-ci.yaml

@@ -13,13 +13,29 @@ on:
   workflow_dispatch: {}
 
 jobs:
-  # JOB 1: Sets up the environment and builds the Docker image needed for all tests.
+  # JOB 1: (NEW) Discovers an available runner and locks it for all subsequent jobs.
+  discover_runner:
+    runs-on: hourly-ci # Picks any available runner from the 'hourly-ci' pool
+    outputs:
+      runner_name: ${{ steps.get_name.outputs.name }}
+    steps:
+      - name: Get runner name
+        id: get_name
+        # This command gets the unique name of the runner (e.g., "my-runner-123")
+        # and saves it as an output variable
+        run: |
+          echo "This workflow will run on: ${{ runner.name }}"
+          echo "name=${{ runner.name }}" >> "$GITHUB_OUTPUT"
+
+  # JOB 2: (UPDATED) Sets up the environment and builds the Docker image.
   setup_and_build:
     if: |
       github.event_name == 'workflow_dispatch' ||
       github.ref == 'refs/heads/main'
-    runs-on: hourly-ci
-    needs: discover_tests
+    # <-- UPDATED: Now needs 'discover_tests' AND 'discover_runner'
+    needs: [discover_tests, discover_runner]
+    # <-- UPDATED: Runs on the specific runner from the discover_runner job
+    runs-on: ${{ needs.discover_runner.outputs.runner_name }}
     permissions:
       contents: read # Required to checkout code and read history
     outputs:
@@ -103,9 +119,12 @@ jobs:
           EOF
           echo "Docker image built successfully."
 
+  # JOB 3: (UPDATED)
   run_unit_tests:
-    needs: setup_and_build
-    runs-on: hourly-ci
+    # <-- UPDATED: Now needs 'setup_and_build' AND 'discover_runner'
+    needs: [setup_and_build, discover_runner]
+    # <-- UPDATED: Runs on the specific runner
+    runs-on: ${{ needs.discover_runner.outputs.runner_name }}
     steps:
       - name: Run pytest in tests/unit_tests
         run: |
@@ -126,8 +145,12 @@ jobs:
           EXITCODE=$?
           echo "Test script exited with code: $EXITCODE"
 
+  # JOB 4: (UPDATED)
   discover_tests:
-    runs-on: hourly-ci
+    # <-- UPDATED: Now needs 'discover_runner'
+    needs: discover_runner
+    # <-- UPDATED: Runs on the specific runner
+    runs-on: ${{ needs.discover_runner.outputs.runner_name }}
     outputs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
     steps:
@@ -140,9 +163,9 @@ jobs:
           # naming convention, excluding the main 'run_all_tests' function itself.
           # The final list is formatted into a JSON array required for the matrix strategy.
           TEST_FUNCTIONS=$( grep '^run_' ./tests/full_tests/ci_gsm8k_tests.sh | \
-                   awk '{print $1}' | \
-                   sed 's/()//' | \
-                   jq -R . | jq -s -c . )
+                      awk '{print $1}' | \
+                      sed 's/()//' | \
+                      jq -R . | jq -s -c . )
 
           echo "Discovered test matrix: $TEST_FUNCTIONS"
           # Fail the job if no tests were found.
@@ -152,9 +175,12 @@ jobs:
           fi
           echo "matrix=$TEST_FUNCTIONS" >> "$GITHUB_OUTPUT"
 
+  # JOB 5: (UPDATED)
   e2e:
-    needs: [setup_and_build, discover_tests]
-    runs-on: hourly-ci
+    # <-- UPDATED: Now needs 'setup_and_build', 'discover_tests', AND 'discover_runner'
+    needs: [setup_and_build, discover_tests, discover_runner]
+    # <-- UPDATED: Runs on the specific runner
+    runs-on: ${{ needs.discover_runner.outputs.runner_name }}
     strategy:
       fail-fast: false
       matrix:
@@ -183,9 +209,12 @@ jobs:
           EXITCODE=$?
           echo "Test script exited with code: $EXITCODE"
 
+  # JOB 6: (UPDATED)
   run_data_parallel_test:
-    needs: setup_and_build
-    runs-on: hourly-ci
+    # <-- UPDATED: Now needs 'setup_and_build' AND 'discover_runner'
+    needs: [setup_and_build, discover_runner]
+    # <-- UPDATED: Runs on the specific runner
+    runs-on: ${{ needs.discover_runner.outputs.runner_name }}
     steps:
       - name: Run Data Parallel test
         run: |
@@ -209,9 +238,12 @@ jobs:
           EXITCODE=$?
           echo "Test script exited with code: $EXITCODE"
 
+  # JOB 7: (UPDATED)
   run_pd_disaggregate_test:
-    needs: setup_and_build
-    runs-on: hourly-ci
+    # <-- UPDATED: Now needs 'setup_and_build' AND 'discover_runner'
+    needs: [setup_and_build, discover_runner]
+    # <-- UPDATED: Runs on the specific runner
+    runs-on: ${{ needs.discover_runner.outputs.runner_name }}
     steps:
       - name: Run PD disaggregate test
         run: |
@@ -236,9 +268,12 @@ jobs:
           EXITCODE=$?
           echo "Test script exited with code: $EXITCODE"
   
+  # JOB 8: (UPDATED)
   store_last_stable_vllm_commit:
-    needs: [setup_and_build, run_unit_tests, e2e, run_data_parallel_test, run_pd_disaggregate_test]
-    runs-on: hourly-ci
+    # <-- UPDATED: Now needs all test jobs AND 'discover_runner'
+    needs: [setup_and_build, run_unit_tests, e2e, run_data_parallel_test, run_pd_disaggregate_test, discover_runner]
+    # <-- UPDATED: Runs on the specific runner
+    runs-on: ${{ needs.discover_runner.outputs.runner_name }}
     permissions:
       contents: write # Permission is required to push a commit
     steps: