Modify benchmark scripts and CI workflow to enable gdb_mode while debugging benchmarks

Author: luszczewskakasia1

.github/workflows/sycl-linux-run-tests.yml

@@ -134,6 +134,10 @@ on:
         type: string
         default: 'false'
         required: False
+      benchmark_gdb_mode:
+        type: string
+        required: False
+        default: 'false'
 
       in_workflow_call_mode:
         description: |
@@ -378,4 +382,6 @@ jobs:
         dry_run: ${{ inputs.benchmark_dry_run }}
         exit_on_failure: ${{ inputs.benchmark_exit_on_failure }}
         build_ref: ${{ inputs.repo_ref }}
-        runner: ${{ inputs.runner }}
+        gdb_mode: ${{ inputs.benchmark_gdb_mode }}
+      env:
+        RUNNERbenchmark__TAG: ${{ inputs.runner }}

.github/workflows/sycl-ur-perf-benchmarking.yml

@@ -78,6 +78,12 @@ on:
         options:
           - 'level_zero:gpu'
           - 'level_zero_v2:gpu'
+      gdb_mode:
+        type: choice
+        options:
+          - false
+          - true
+        default: false
 
 concurrency:
   # Cancel a currently running workflow for:
@@ -175,7 +181,7 @@ jobs:
     uses: ./.github/workflows/sycl-linux-run-tests.yml
     secrets: inherit
     with:
-      name: "Benchmarks (${{ matrix.runner }}, ${{ matrix.backend }}, preset: ${{ inputs.preset }})"
+      name: "Benchmarks (${{ matrix.runner }}, ${{ matrix.backend }}, preset: ${{ inputs.preset }}, gdb_mode: ${{ inputs.gdb_mode }})"
       runner: ${{ matrix.runner }}
       image: ghcr.io/intel/llvm/sycl_ubuntu2404_nightly:latest
       image_options: -u 1001 --device=/dev/dri -v /dev/dri/by-path:/dev/dri/by-path --privileged --cap-add SYS_ADMIN
@@ -185,6 +191,7 @@ jobs:
       benchmark_save_name: ${{ needs.sanitize_inputs_dispatch.outputs.benchmark_save_name }}
       benchmark_preset: ${{ inputs.preset }}
       benchmark_exit_on_failure: ${{ inputs.exit_on_failure }}
+      benchmark_gdb_mode: ${{ inputs.gdb_mode }}
       repo_ref: ${{ needs.sanitize_inputs_dispatch.outputs.build_ref }}
       toolchain_artifact: ${{ needs.build_sycl_dispatch.outputs.toolchain_artifact }}
       toolchain_artifact_filename: ${{ needs.build_sycl_dispatch.outputs.toolchain_artifact_filename }}
@@ -233,6 +240,7 @@ jobs:
       benchmark_upload_results: true
       benchmark_save_name: ${{ matrix.save_name }}
       benchmark_preset: ${{ matrix.preset }}
+      benchmark_gdb_mode: false
       repo_ref: ${{ matrix.ref }}
       toolchain_artifact: ${{ needs.build_nightly.outputs.toolchain_artifact }}
       toolchain_artifact_filename: ${{ needs.build_nightly.outputs.toolchain_artifact_filename }}
@@ -259,5 +267,6 @@ jobs:
       benchmark_preset: 'Minimal'
       benchmark_dry_run: true
       benchmark_exit_on_failure: true
+      benchmark_gdb_mode: ${{ inputs.gdb_mode }}
       repo_ref: ${{ github.sha }}
   # END benchmark framework builds and runs on PRs path

devops/actions/run-tests/benchmark/action.yml

@@ -23,6 +23,9 @@ inputs:
     type: string
     required: False
     default: ""
+  gdb_mode:
+    type: string
+    required: False
   # dry-run is passed only to compare.py (to not fail on regression), not to main.py (where such flag would omit all benchmark runs)
   dry_run:
     type: string
@@ -267,6 +270,7 @@ runs:
     env:
       BENCH_WORKDIR: ${{ steps.establish_outputs.outputs.BENCH_WORKDIR }}
       BENCHMARK_RESULTS_REPO_PATH: ${{ steps.establish_outputs.outputs.BENCHMARK_RESULTS_REPO_PATH }}
+      LLVM_BENCHMARKS_USE_GDB: ${{ inputs.gdb_mode }}
     run: |
       # Build and run benchmarks
 
@@ -280,6 +284,7 @@ runs:
       echo "::endgroup::"
       echo "::group::run_benchmarks"
 
+      export LLVM_BENCHMARKS_USE_GDB=${LLVM_BENCHMARKS_USE_GDB}
       numactl --cpunodebind "$NUMA_NODE" --membind "$NUMA_NODE" \
       ./devops/scripts/benchmarks/main.py "$BENCH_WORKDIR" \
         --sycl "$(realpath $CMPLR_ROOT)" \
@@ -299,17 +304,21 @@ runs:
       echo "::endgroup::"
       echo "::group::compare_results"
 
-      python3 ./devops/scripts/benchmarks/compare.py to_hist \
-        --avg-type EWMA \
-        --cutoff "$(date -u -d '7 days ago' +'%Y%m%d_%H%M%S')" \
-        --name "$SAVE_NAME" \
-        --compare-file "${BENCHMARK_RESULTS_REPO_PATH}/results/${SAVE_NAME}_${SAVE_TIMESTAMP}.json" \
-        --results-dir "${BENCHMARK_RESULTS_REPO_PATH}/results/" \
-        --regression-filter '^[a-z_]+_sycl .* CPU count' \
-        --regression-filter-type 'SYCL benchmark (measured using CPU cycle count)' \
-        --verbose \
-        --produce-github-summary \
-        ${{ inputs.dry_run == 'true' && '--dry-run' || '' }} \
+      if [ "$LLVM_BENCHMARKS_USE_GDB" == "false" ]; then
+        python3 ./devops/scripts/benchmarks/compare.py to_hist \
+          --avg-type EWMA \
+          --cutoff "$(date -u -d '7 days ago' +'%Y%m%d_%H%M%S')" \
+          --name "$SAVE_NAME" \
+          --compare-file "${BENCHMARK_RESULTS_REPO_PATH}/results/${SAVE_NAME}_${SAVE_TIMESTAMP}.json" \
+          --results-dir "${BENCHMARK_RESULTS_REPO_PATH}/results/" \
+          --regression-filter '^[a-z_]+_sycl .* CPU count' \
+          --regression-filter-type 'SYCL benchmark (measured using CPU cycle count)' \
+          --verbose \
+          --produce-github-summary \
+          ${{ inputs.dry_run == 'true' && '--dry-run' || '' }}
+      else
+        echo "Skipping regression comparison due to GDB mode enabled."
+      fi
 
       echo "::endgroup::"
   - name: Run benchmarks integration tests

devops/scripts/benchmarks/benches/compute.py

@@ -6,6 +6,7 @@
 import csv
 import io
 import math
+import os
 from enum import Enum
 from itertools import product
 from pathlib import Path
@@ -76,6 +77,7 @@ def setup(self) -> None:
                 "compute-benchmarks",
                 use_installdir=False,
             )
+        print(self._project)
 
         if not self._project.needs_rebuild():
             log.info(f"Rebuilding {self._project.name} skipped")
@@ -88,7 +90,15 @@ def setup(self) -> None:
             f"-DUSE_SYSTEM_LEVEL_ZERO=OFF",
             f"-DCMAKE_CXX_COMPILER=clang++",
             f"-DCMAKE_C_COMPILER=clang",
+
         ]
+        
+        is_gdb_mode = os.environ.get("LLVM_BENCHMARKS_USE_GDB", "") == "true"
+        if is_gdb_mode:
+            extra_args += [
+                f"-DCMAKE_CXX_FLAGS_RELWITHDEBINFO:STRING=-O2 -g -DNDEBUG -fdebug-info-for-profiling",
+            ]
+        
         if options.ur_adapter == "cuda":
             extra_args += [
                 "-DBUILD_SYCL_WITH_CUDA=ON",
@@ -561,6 +571,12 @@ def __enabled_runtimes(self) -> list[RUNTIMES]:
         return runtimes
 
     def __parse_output(self, output: str) -> list[tuple[float, float]]:
+        is_gdb_mode = os.environ.get("LLVM_BENCHMARKS_USE_GDB", "") == "true"
+        print(output)
+
+        if is_gdb_mode:
+            return [(0.0, 0.0)]
+
         csv_file = io.StringIO(output)
         reader = csv.reader(csv_file)
         next(reader, None)

devops/scripts/benchmarks/git_project.py

@@ -74,11 +74,15 @@ def configure(
         add_sycl: bool = False,
     ) -> None:
         """Configures the project."""
+
+        is_gdb_mode = os.environ.get("LLVM_BENCHMARKS_USE_GDB", "") == "true"
+        build_type = "RelWithDebInfo" if is_gdb_mode else "Release"
+        
         cmd = [
             "cmake",
             f"-S {self.src_dir}",
             f"-B {self.build_dir}",
-            f"-DCMAKE_BUILD_TYPE=Release",
+            f"-DCMAKE_BUILD_TYPE={build_type}",
         ]
         if self._use_installdir:
             cmd.append(f"-DCMAKE_INSTALL_PREFIX={self.install_dir}")

devops/scripts/benchmarks/utils/utils.py

@@ -48,6 +48,25 @@ def run(
         if isinstance(command, str):
             command = command.split()
 
+        is_gdb_mode = os.environ.get("LLVM_BENCHMARKS_USE_GDB", "") == "true"
+        if command[0] == "taskset" and is_gdb_mode:
+            gdb_cmd = [
+                "gdb",
+                "-return-child-result",
+                "--batch",
+                "--ex",
+                "set confirm off",
+                "--ex",
+                "run",
+                "--ex",
+                "bt",
+                "--ex",
+                "quit",
+                "--args",
+            ] + command
+            command = gdb_cmd
+            print(f"Running in gdb mode: {command}")
+
         env = os.environ.copy()
         for ldlib in ld_library:
             if os.path.isdir(ldlib):