Modify benchmark scripts and CI workflow to enable gdb_mode while debugging benchmarks

Author: luszczewskakasia1

.github/workflows/sycl-linux-run-tests.yml

@@ -134,6 +134,10 @@ on:
         type: string
         default: 'false'
         required: False
+      benchmark_gdb_mode:
+        type: string
+        required: False
+        default: 'false'
 
       in_workflow_call_mode:
         description: |
@@ -379,3 +383,4 @@ jobs:
         exit_on_failure: ${{ inputs.benchmark_exit_on_failure }}
         build_ref: ${{ inputs.repo_ref }}
         runner: ${{ inputs.runner }}
+        gdb_mode: ${{ inputs.benchmark_gdb_mode }}

.github/workflows/sycl-ur-perf-benchmarking.yml

@@ -78,6 +78,11 @@ on:
         options:
           - 'level_zero:gpu'
           - 'level_zero_v2:gpu'
+      gdb_mode:
+        type: choice
+        options:
+          - false
+          - true
 
 concurrency:
   # Cancel a currently running workflow for:
@@ -175,7 +180,7 @@ jobs:
     uses: ./.github/workflows/sycl-linux-run-tests.yml
     secrets: inherit
     with:
-      name: "Benchmarks (${{ matrix.runner }}, ${{ matrix.backend }}, preset: ${{ inputs.preset }})"
+      name: "Benchmarks (${{ matrix.runner }}, ${{ matrix.backend }}, preset: ${{ inputs.preset }}, gdb_mode: ${{ inputs.gdb_mode }})"
       runner: ${{ matrix.runner }}
       image: ghcr.io/intel/llvm/sycl_ubuntu2404_nightly:latest
       image_options: -u 1001 --device=/dev/dri -v /dev/dri/by-path:/dev/dri/by-path --privileged --cap-add SYS_ADMIN
@@ -186,6 +191,7 @@ jobs:
       benchmark_preset: ${{ inputs.preset }}
       benchmark_exit_on_failure: ${{ inputs.exit_on_failure }}
       repo_ref: ${{ needs.sanitize_inputs_dispatch.outputs.build_ref }}
+      gdb_mode: ${{ inputs.gdb_mode }}
       toolchain_artifact: ${{ needs.build_sycl_dispatch.outputs.toolchain_artifact }}
       toolchain_artifact_filename: ${{ needs.build_sycl_dispatch.outputs.toolchain_artifact_filename }}
       toolchain_decompress_command: ${{ needs.build_sycl_dispatch.outputs.toolchain_decompress_command }}
@@ -234,6 +240,7 @@ jobs:
       benchmark_save_name: ${{ matrix.save_name }}
       benchmark_preset: ${{ matrix.preset }}
       repo_ref: ${{ matrix.ref }}
+      gdb_mode: false
       toolchain_artifact: ${{ needs.build_nightly.outputs.toolchain_artifact }}
       toolchain_artifact_filename: ${{ needs.build_nightly.outputs.toolchain_artifact_filename }}
       toolchain_decompress_command: ${{ needs.build_nightly.outputs.toolchain_decompress_command }}
@@ -260,4 +267,5 @@ jobs:
       benchmark_dry_run: true
       benchmark_exit_on_failure: true
       repo_ref: ${{ github.sha }}
+      gdb_mode: ${{ inputs.gdb_mode }}
   # END benchmark framework builds and runs on PRs path

devops/actions/run-tests/benchmark/action.yml

@@ -23,6 +23,9 @@ inputs:
     type: string
     required: False
     default: ""
+  gdb_mode:
+    type: string
+    required: False
   # dry-run is passed only to compare.py (to not fail on regression), not to main.py (where such flag would omit all benchmark runs)
   dry_run:
     type: string
@@ -267,6 +270,7 @@ runs:
     env:
       BENCH_WORKDIR: ${{ steps.establish_outputs.outputs.BENCH_WORKDIR }}
       BENCHMARK_RESULTS_REPO_PATH: ${{ steps.establish_outputs.outputs.BENCHMARK_RESULTS_REPO_PATH }}
+      LLVM_BENCHMARKS_USE_GDB: ${{ inputs.gdb_mode }}
     run: |
       # Build and run benchmarks
 
@@ -280,6 +284,7 @@ runs:
       echo "::endgroup::"
       echo "::group::run_benchmarks"
 
+      export LLVM_BENCHMARKS_USE_GDB=${LLVM_BENCHMARKS_USE_GDB}
       numactl --cpunodebind "$NUMA_NODE" --membind "$NUMA_NODE" \
       ./devops/scripts/benchmarks/main.py "$BENCH_WORKDIR" \
         --sycl "$(realpath $CMPLR_ROOT)" \
@@ -299,17 +304,21 @@ runs:
       echo "::endgroup::"
       echo "::group::compare_results"
 
-      python3 ./devops/scripts/benchmarks/compare.py to_hist \
-        --avg-type EWMA \
-        --cutoff "$(date -u -d '7 days ago' +'%Y%m%d_%H%M%S')" \
-        --name "$SAVE_NAME" \
-        --compare-file "${BENCHMARK_RESULTS_REPO_PATH}/results/${SAVE_NAME}_${SAVE_TIMESTAMP}.json" \
-        --results-dir "${BENCHMARK_RESULTS_REPO_PATH}/results/" \
-        --regression-filter '^[a-z_]+_sycl .* CPU count' \
-        --regression-filter-type 'SYCL benchmark (measured using CPU cycle count)' \
-        --verbose \
-        --produce-github-summary \
-        ${{ inputs.dry_run == 'true' && '--dry-run' || '' }} \
+      if [ "$LLVM_BENCHMARKS_USE_GDB" == "false" ]; then
+        python3 ./devops/scripts/benchmarks/compare.py to_hist \
+          --avg-type EWMA \
+          --cutoff "$(date -u -d '7 days ago' +'%Y%m%d_%H%M%S')" \
+          --name "$SAVE_NAME" \
+          --compare-file "${BENCHMARK_RESULTS_REPO_PATH}/results/${SAVE_NAME}_${SAVE_TIMESTAMP}.json" \
+          --results-dir "${BENCHMARK_RESULTS_REPO_PATH}/results/" \
+          --regression-filter '^[a-z_]+_sycl .* CPU count' \
+          --regression-filter-type 'SYCL benchmark (measured using CPU cycle count)' \
+          --verbose \
+          --produce-github-summary \
+          ${{ inputs.dry_run == 'true' && '--dry-run' || '' }}
+      else
+        echo "Skipping regression comparison due to GDB mode enabled."
+      fi
 
       echo "::endgroup::"
   - name: Run benchmarks integration tests

devops/scripts/benchmarks/benches/compute.py

@@ -6,6 +6,7 @@
 import csv
 import io
 import math
+import os
 from enum import Enum
 from itertools import product
 from pathlib import Path
@@ -88,6 +89,8 @@ def setup(self) -> None:
             f"-DUSE_SYSTEM_LEVEL_ZERO=OFF",
             f"-DCMAKE_CXX_COMPILER=clang++",
             f"-DCMAKE_C_COMPILER=clang",
+            f"-DCMAKE_CXX_FLAGS_RELWITHDEBINFO:STRING=-O2 -g -DNDEBUG -fdebug-info-for-profiling",
+            f"-DCMAKE_BUILD_TYPE=RelWithDebInfo",
         ]
         if options.ur_adapter == "cuda":
             extra_args += [
@@ -561,28 +564,36 @@ def __enabled_runtimes(self) -> list[RUNTIMES]:
         return runtimes
 
     def __parse_output(self, output: str) -> list[tuple[float, float]]:
-        csv_file = io.StringIO(output)
-        reader = csv.reader(csv_file)
-        next(reader, None)
-        results = []
-        while True:
-            data_row = next(reader, None)
-            if data_row is None:
-                break
-            try:
-                mean = float(data_row[1])
-                median = float(data_row[2])
-                # compute benchmarks report stddev as %
-                stddev = mean * (float(data_row[3].strip("%")) / 100.0)
-                if not math.isfinite(stddev):
-                    stddev = 0.0  # Default to 0.0 if stddev is invalid
-
-                results.append((median, stddev))
-            except (ValueError, IndexError) as e:
-                raise ValueError(f"Error parsing output: {e}")
-        if len(results) == 0:
-            raise ValueError("Benchmark output does not contain data.")
-        return results
+        is_gdb_benchmarks = os.environ.get("LLVM_BENCHMARKS_USE_GDB", "") == "true"
+        print(output)
+
+        if is_gdb_benchmarks:
+            return [(0.0, 0.0)]
+
+        else:
+            csv_file = io.StringIO(output)
+            reader = csv.reader(csv_file)
+            next(reader, None)
+            results = []
+            while True:
+                data_row = next(reader, None)
+                # print("my data",data_row)
+                if data_row is None:
+                    break
+                try:
+                    mean = float(data_row[1])
+                    median = float(data_row[2])
+                    # compute benchmarks report stddev as %
+                    stddev = mean * (float(data_row[3].strip("%")) / 100.0)
+                    if not math.isfinite(stddev):
+                        stddev = 0.0  # Default to 0.0 if stddev is invalid
+
+                    results.append((median, stddev))
+                except (ValueError, IndexError) as e:
+                    raise ValueError(f"Error parsing output: {e}")
+            if len(results) == 0:
+                raise ValueError("Benchmark output does not contain data.")
+            return results
 
     def __validate_attr(self, attr_name: str):
         if (

devops/scripts/benchmarks/utils/utils.py

@@ -48,6 +48,25 @@ def run(
         if isinstance(command, str):
             command = command.split()
 
+        is_gdb_benchmarks = os.environ.get("LLVM_BENCHMARKS_USE_GDB", "") == "true"
+        if command[0] == "taskset" and is_gdb_benchmarks:
+            gdb_cmd = [
+                "gdb",
+                "-return-child-result",
+                "--batch",
+                "--ex",
+                "set confirm off",
+                "--ex",
+                "run",
+                "--ex",
+                "bt",
+                "--ex",
+                "quit",
+                "--args",
+            ] + command
+            command = gdb_cmd
+            print(f"Running in gdb mode: {command}")
+
         env = os.environ.copy()
         for ldlib in ld_library:
             if os.path.isdir(ldlib):