sycl-ur-perf-benchmarking.yml

# A combined workflow for all benchmarks-related jobs for SYCL and UR.
# Supports both manual triggering (dispatch) and nightly runs.
# It also tests changes to benchmark scripts/framework in PR, if modified.
name: SYCL Run Benchmarks

on:
  schedule:
    # 2 hours after SYCL nightly
    - cron: '0 5 * * *'
  # Run on pull requests only when a benchmark-related files were changed.
  pull_request:
    # These paths are exactly the same as in sycl-linux/windows-precommit.yml (to ignore over there)
    paths:
      - 'devops/scripts/benchmarks/**'
      - 'devops/actions/run-tests/benchmark/**'
      - '.github/workflows/sycl-ur-perf-benchmarking.yml'
  workflow_dispatch:
    inputs:
      preset:
        type: choice
        description: |
          Benchmark presets to run, See /devops/scripts/benchmarks/presets.py. Hint: Minimal is compute-benchmarks only.
        options:
          - Full
          - SYCL
          - Minimal
          - Core
          - Normal
          - Test
          - Gromacs
          - OneDNN
        default: 'Minimal'  # Only compute-benchmarks
      pr_no:
        type: string
        description: |
          PR no. to build SYCL from - it will be built from HEAD of incoming branch.

          Leave both pr_no and commit_hash empty to use the latest commit from branch/tag this workflow started from.
        required: false
        default: ''
      commit_hash:
        type: string
        description: |
          Commit hash (within intel/llvm) to build SYCL from.

          Leave both pr_no and commit_hash empty to use the latest commit from branch/tag this workflow started from.
        required: false
        default: ''
      save_name:
        type: string
        description: |
          Name to use for the benchmark result
        required: false
        default: ''
      upload_results:
        description: Save and upload results (to https://intel.github.io/llvm/benchmarks)
        type: choice
        options:
          - false
          - true
        default: true
      exit_on_failure:
        description: Fail benchmark script on any error. Limit number of iterations to just test correctness.
        type: choice
        options:
          - false
          - true
        default: false
      runner:
        description: Self-hosted runner to use for the benchmarks
        type: choice
        options:
          - '["PVC_PERF"]'
          - '["BMG_PERF"]'
      backend:
        description: Backend to use
        type: choice
        options:
          - 'level_zero:gpu'
          - 'level_zero_v2:gpu'

concurrency:
  # Cancel a currently running workflow for:
  # - the same PR
  # - manual dispatch if pr_no or commit_hash is repeated, and the runner & backend are the same
  # - scheduled runs if they overlap in time
  group: >-
    ${{ github.workflow }}-${{ github.event_name }}-
      ${{ github.event.pull_request.number || inputs.pr_no || inputs.commit_hash || github.ref }}-
      ${{ inputs.runner }}-${{ inputs.backend }}
  cancel-in-progress: true

permissions: read-all

jobs:
  # Manual trigger (dispatch) path:
  sanitize_inputs_dispatch:
    name: '[Dispatch] Sanitize inputs'
    if: github.event_name == 'workflow_dispatch'
    runs-on: ubuntu-latest
    env:
      COMMIT_HASH: ${{ inputs.commit_hash }}
      PR_NO: ${{ inputs.pr_no }}
      SAVE_NAME: ${{ inputs.save_name }}
    outputs:
      benchmark_save_name: ${{ steps.sanitize.outputs.benchmark_save_name }}
      build_ref: ${{ steps.sanitize.outputs.build_ref }}
    steps:
      - id: sanitize
        run: |
          # Validate user inputs:
          # usage: check_if_nonempty <var> <regex to check var against> <err message>
          check_nonempty() {
            [ -z "$1" ] && return
            if [ -z "$(echo "$1" | grep -P "$2")" ]; then
              echo "$3"
              exit 1
            fi
          }
          check_nonempty "$COMMIT_HASH" '^[0-9a-f]{7,}$' "Bad commit hash (or hash short)."
          check_nonempty "$PR_NO" '^[0-9]+$' "Bad PR number."
          check_nonempty "$SAVE_NAME" '^[A-Za-z][A-Za-z0-9_-]+$' "Bad save name."

          BENCHMARK_SAVE_NAME=""
          BUILD_REF="${{ github.ref }}"
          if [ -n "$SAVE_NAME" ]; then
            BENCHMARK_SAVE_NAME="$(echo "$SAVE_NAME" | tr -cd 'A-Za-z0-9_-')"
          fi;
          if [ -n "$COMMIT_HASH" ]; then
            echo "Using commit hash $COMMIT_HASH for build..."
            BUILD_REF="$COMMIT_HASH"
            shortened_commit="$(echo "$COMMIT_HASH" | cut -c 1-7)"
            [ -z "$BENCHMARK_SAVE_NAME" ] && BENCHMARK_SAVE_NAME="Commit_${shortened_commit}"
          elif [ -n "$PR_NO" ]; then
            echo "Using PR no. $PR_NO for build..."
            BUILD_REF="refs/pull/$PR_NO/head"
            [ -z "$BENCHMARK_SAVE_NAME" ] && BENCHMARK_SAVE_NAME="PR_${PR_NO}"
          fi
          [ -z "$BENCHMARK_SAVE_NAME" ] && BENCHMARK_SAVE_NAME="Baseline"

          echo "benchmark_save_name=$BENCHMARK_SAVE_NAME" >> $GITHUB_OUTPUT
          echo "build_ref=$BUILD_REF" >> $GITHUB_OUTPUT

          echo "Final sanitized values:"
          cat $GITHUB_OUTPUT

  build_sycl_dispatch:
    name: '[Dispatch] Build SYCL'
    needs: [ sanitize_inputs_dispatch ]
    uses: ./.github/workflows/sycl-linux-build.yml
    with:
      build_ref: ${{ needs.sanitize_inputs_dispatch.outputs.build_ref }}
      build_cache_root: "/__w/"
      build_cache_suffix: "prod_noassert"
      build_configure_extra_args: "--no-assertions"
      build_image: "ghcr.io/intel/llvm/sycl_ubuntu2404_nightly:latest"
      cc: clang
      cxx: clang++
      changes: '[]'
      toolchain_artifact: sycl_linux_prod_noassert

  benchmark_dispatch:
    name: '[Dispatch] Benchmarks'
    needs: [ build_sycl_dispatch, sanitize_inputs_dispatch ]
    permissions:
      contents: write
      packages: read
    if: ${{ !cancelled() && needs.build_sycl_dispatch.outputs.build_conclusion == 'success' }}
    strategy:
      matrix:
        include:
          # Set default values if not specified:
          - runner: ${{ inputs.runner || '["PVC_PERF"]' }}
            backend: ${{ inputs.backend || 'level_zero:gpu' }}
    uses: ./.github/workflows/sycl-linux-run-tests.yml
    secrets: inherit
    with:
      name: "Benchmarks (${{ matrix.runner }}, ${{ matrix.backend }}, preset: ${{ inputs.preset }})"
      runner: ${{ matrix.runner }}
      image: ghcr.io/intel/llvm/sycl_ubuntu2404_nightly:latest
      image_options: -u 1001 --device=/dev/dri -v /dev/dri/by-path:/dev/dri/by-path --privileged --cap-add SYS_ADMIN
      target_devices: ${{ matrix.backend }}
      tests_selector: benchmarks
      benchmark_upload_results: ${{ inputs.upload_results }}
      benchmark_save_name: ${{ needs.sanitize_inputs_dispatch.outputs.benchmark_save_name }}
      benchmark_preset: ${{ inputs.preset }}
      benchmark_exit_on_failure: ${{ inputs.exit_on_failure }}
      repo_ref: ${{ needs.sanitize_inputs_dispatch.outputs.build_ref }}
      toolchain_artifact: ${{ needs.build_sycl_dispatch.outputs.toolchain_artifact }}
      toolchain_artifact_filename: ${{ needs.build_sycl_dispatch.outputs.toolchain_artifact_filename }}
      toolchain_decompress_command: ${{ needs.build_sycl_dispatch.outputs.toolchain_decompress_command }}
  # END manual trigger (dispatch) path

  # Nightly benchmarking path:
  build_nightly:
    name: '[Nightly] Build SYCL'
    if: github.repository == 'intel/llvm' && github.event_name == 'schedule'
    uses: ./.github/workflows/sycl-linux-build.yml
    secrets: inherit
    with:
      build_cache_root: "/__w/"
      build_configure_extra_args: '--no-assertions'
      build_image: ghcr.io/intel/llvm/ubuntu2404_build:latest

      toolchain_artifact: sycl_linux_default
      toolchain_artifact_filename: sycl_linux.tar.gz

  benchmark_nightly:
    name: '[Nightly] Benchmarks'
    needs: [build_nightly]
    permissions:
      contents: write
      packages: read
    if: ${{ !cancelled() && needs.build_nightly.outputs.build_conclusion == 'success' }}
    strategy:
      fail-fast: false
      matrix:
        runner: ['["PVC_PERF"]', '["BMG_PERF"]']
        backend: ['level_zero:gpu', 'level_zero_v2:gpu']
        include:
          - ref: ${{ github.sha }}
            save_name: 'Baseline'
            preset: 'Full'
    uses: ./.github/workflows/sycl-linux-run-tests.yml
    secrets: inherit
    with:
      name: "Benchmarks (${{ matrix.runner }}, ${{ matrix.backend }}, preset: ${{ matrix.preset }})"
      runner: ${{ matrix.runner }}
      image: ghcr.io/intel/llvm/sycl_ubuntu2404_nightly:latest
      image_options: -u 1001 --device=/dev/dri -v /dev/dri/by-path:/dev/dri/by-path --privileged --cap-add SYS_ADMIN
      target_devices: ${{ matrix.backend }}
      tests_selector: benchmarks
      benchmark_upload_results: true
      benchmark_save_name: ${{ matrix.save_name }}
      benchmark_preset: ${{ matrix.preset }}
      repo_ref: ${{ matrix.ref }}
      toolchain_artifact: ${{ needs.build_nightly.outputs.toolchain_artifact }}
      toolchain_artifact_filename: ${{ needs.build_nightly.outputs.toolchain_artifact_filename }}
      toolchain_decompress_command: ${{ needs.build_nightly.outputs.toolchain_decompress_command }}
  # END nightly benchmarking path

  # BEGIN benchmark framework builds and runs on PRs path
  # TODO: When we have stable BMG runner(s), consider moving this job to that runner.
  test_benchmark_framework:
    name: '[PR] Benchmark suite testing'
    permissions:
      contents: write
      packages: read
    if: github.event_name == 'pull_request'
    uses: ./.github/workflows/sycl-linux-run-tests.yml
    with:
      name: 'Framework test: PVC_PERF, L0, Minimal preset'
      runner: '["PVC_PERF"]'
      image: ghcr.io/intel/llvm/sycl_ubuntu2404_nightly:latest
      image_options: -u 1001 --device=/dev/dri -v /dev/dri/by-path:/dev/dri/by-path --privileged --cap-add SYS_ADMIN
      target_devices: 'level_zero:gpu'
      tests_selector: benchmarks
      benchmark_upload_results: false
      benchmark_preset: 'Minimal'
      benchmark_dry_run: true
      benchmark_exit_on_failure: true
      repo_ref: ${{ github.sha }}
  # END benchmark framework builds and runs on PRs path