Skip to content

Commit 9da3ef2

Browse files
committed
Merge branch 'main' into test_fused_atten_prefetch
2 parents d306ec9 + 33e5e68 commit 9da3ef2

21 files changed

Lines changed: 648 additions & 405 deletions

‎examples/KernelBench/test-kernel-bench.py‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
1-
# RUN: python %s --ci | FileCheck %s
2-
# RUN: python %s --ci --no-torch-compile | FileCheck %s
1+
# RUN: %PYTHON %s --ci | FileCheck %s
2+
# RUN: %PYTHON %s --ci --no-torch-compile | FileCheck %s
33

44
# REQUIRES: torch
5+
# REQUIRES: torch_mlir
56
# REQUIRES: kernel_bench
67

78
import argparse
@@ -10,12 +11,17 @@
1011
from pathlib import Path
1112

1213
import yaml
14+
import sys
1315

1416
from lighthouse.execution.target import TargetInfo
1517
from lighthouse.pipeline import find_pipeline_file
1618

17-
script_path = Path(__file__).parent
19+
script_path = Path(__file__).resolve().parent
1820
project_root = script_path.parent.parent
21+
if str(project_root) not in sys.path:
22+
sys.path.insert(0, str(project_root))
23+
24+
1925
kb_program = project_root / "tools" / "kernel-bench"
2026
kb_path = project_root / "third_party" / "KernelBench" / "KernelBench"
2127
yaml_files = [
@@ -165,6 +171,7 @@ def get_flops_per_second(stdout: str, gflops: float) -> float:
165171
command_line = []
166172

167173
command_line += [
174+
sys.executable,
168175
str(kb_program),
169176
str(kb_kernel),
170177
"--dtype",

‎examples/xegpu/kernel_bench.py‎

Lines changed: 40 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,6 @@
5252
import torch
5353
import torch._dynamo as dynamo
5454
import os
55-
import json
5655
import numpy as np
5756
from mlir import ir
5857

@@ -62,6 +61,7 @@
6261
from lighthouse.utils.mlir import inspect_payload
6362
from lighthouse.execution.runner import Runner
6463
from lighthouse.schedule.xegpu import XeGPUParameterSelector
64+
from lighthouse.schedule.parameters import ScheduleParameters
6565
from lighthouse.pipeline.driver import TransformDriver
6666
from lighthouse.schedule.xegpu import (
6767
mlp_schedule,
@@ -72,11 +72,18 @@
7272
from lighthouse.pipeline.helper import PipelineInterrupt
7373
from lighthouse.ingress.torch import gpu_backend, TargetDialect
7474
from lighthouse.ingress.torch.compile import TorchMemoryManager
75-
from tune_matmul_costmodel import optimize_payload, dump_configs_json
75+
from tune_matmul_costmodel import optimize_payload
7676
from tune_utils import run_with_timeout
7777
from csv_logger import CSVLogger
7878

7979

80+
def dtype_to_torch_dtype(datatype: str) -> torch.dtype:
81+
return {
82+
"f16": torch.float16,
83+
"bf16": torch.bfloat16,
84+
}[datatype]
85+
86+
8087
def inspect_kb_payload(module: ir.Module) -> tuple[str, dict]:
8188
"""
8289
Extract metadata from the payload IR module.
@@ -90,14 +97,16 @@ def inspect_kb_payload(module: ir.Module) -> tuple[str, dict]:
9097
return payload_func_name, func_metadata
9198

9299

93-
def infer_parameters(mod: ir.Module, verbose: int = 0) -> tuple[dict, str, list[dict]]:
100+
def infer_parameters(
101+
mod: ir.Module, verbose: int = 0
102+
) -> tuple[dict, str, ScheduleParameters]:
94103
"""
95104
Inspects payload and selects lowering schedule and tile size parameters.
96105
97106
Returns:
98107
func_metadata: Payload function metadata dict.
99108
schedule kind: Name of selected lowering schedule.
100-
schedule_parameters: List of parameter dicts, one per layer.
109+
schedule_parameters: ScheduleParameters for the lowering schedule.
101110
"""
102111
payload_func_name, func_metadata = inspect_kb_payload(mod)
103112
if verbose > 0:
@@ -155,6 +164,7 @@ def infer_parameters(mod: ir.Module, verbose: int = 0) -> tuple[dict, str, list[
155164

156165
# Use fixed tile sizes for now
157166
layer_params = {
167+
"layer_kind": "elemwise",
158168
"wg_m": 128,
159169
"wg_n": 256,
160170
"sg_m": 32,
@@ -163,7 +173,7 @@ def infer_parameters(mod: ir.Module, verbose: int = 0) -> tuple[dict, str, list[
163173
"load_n": 16,
164174
}
165175
# NOTE assume all elemwise layers will be fused to a single layer
166-
schedule_params = [layer_params]
176+
schedule_params = ScheduleParameters([layer_params])
167177
schedule_kind = "elemwise"
168178
elif len(elemwise) > 0 and len(reduction) > 0:
169179
# elemwise + reduction kernel, e.g. softmax or layer norm
@@ -176,7 +186,7 @@ def infer_parameters(mod: ir.Module, verbose: int = 0) -> tuple[dict, str, list[
176186
write_bytes = int(np.prod(shape)) * res_bytes
177187

178188
layer_params = {
179-
"sizes": shape,
189+
"layer_kind": "reduction",
180190
"wg_rows": 64,
181191
"sg_rows": 8,
182192
"subgroup_size": 16,
@@ -193,7 +203,7 @@ def infer_parameters(mod: ir.Module, verbose: int = 0) -> tuple[dict, str, list[
193203
raise ValueError(
194204
f"Shape {shape} dimension 1 not divisible by reduction_step_size={layer_params['reduction_step_size']}"
195205
)
196-
schedule_params = [layer_params]
206+
schedule_params = ScheduleParameters([layer_params])
197207
schedule_kind = "reduction"
198208
else:
199209
print("Layers:")
@@ -278,7 +288,8 @@ def compile_and_eval(nruns: int, nwarmup: int, **kwparams) -> ir.Module:
278288
final_mod = lower_to_llvm(
279289
copy_module(mod),
280290
schedule_kind="mlp",
281-
schedule_params=[kwparams],
291+
schedule_params=ScheduleParameters([kwparams]),
292+
device="B70",
282293
stop_at_stage=None,
283294
benchmark=True,
284295
payload_func_name=payload_func_name,
@@ -364,35 +375,41 @@ def infer_params_and_lower(
364375
# runtime tuning for matmul kernels
365376
if os.path.isfile(params_cache_json):
366377
print(f"Loading cached parameters from {params_cache_json}")
367-
with open(params_cache_json) as f:
368-
params_dict = json.load(f)
369-
schedule_params = [params_dict]
378+
schedule_params = ScheduleParameters.from_json(filename=params_cache_json)
370379
else:
371380
print(f"Tuning parameters and saving to {params_cache_json}")
381+
# construct a buffer for kernel result
382+
res_type = kernel_metadata["inputs"][-1] # result is last input
383+
shape = res_type.shape
384+
dtype = dtype_to_torch_dtype(str(res_type.element_type))
385+
res_buffer = torch.zeros(shape, dtype=dtype).to("xpu")
386+
kernel_inputs = [*torch_all_inputs, res_buffer]
372387
# tune matmul parameters
373388
configs = tune_matmul_layer(
374389
matmuls[0],
375390
mod,
376391
payload_func_name,
377-
torch_all_inputs,
392+
kernel_inputs,
378393
func_metadata["total_flops"],
379394
func_metadata["read_bytes"],
380395
func_metadata["write_bytes"],
381396
)
382-
schedule_params = [configs[0][1]]
383-
dump_configs_json(
384-
schedule_params[0], filename_prefix=params_cache_json.rsplit(".", 1)[0]
385-
)
397+
schedule_params = ScheduleParameters([configs[0][1]])
398+
schedule_params.to_json(filename=params_cache_json, overwrite=True)
386399
else:
387400
print(f"Using default parameters for {schedule_kind} schedule")
401+
print(f"Saving applied parameters to {params_cache_json}")
402+
schedule_params.to_json(filename=params_cache_json, overwrite=True)
388403

389404
if verbose > 2:
390405
print("Payload module before lowering:")
391406
print(mod)
392407
if verbose > 1:
393408
print(f"Applying '{schedule_kind}' schedule with params:")
394409
param_list = (
395-
schedule_params if isinstance(schedule_params, list) else [schedule_params]
410+
schedule_params
411+
if isinstance(schedule_params, (list, ScheduleParameters))
412+
else [schedule_params]
396413
)
397414
for i, param_dict in enumerate(param_list):
398415
print(f" Parameters for layer {i}:")
@@ -403,6 +420,7 @@ def infer_params_and_lower(
403420
mod=mod,
404421
schedule_kind=schedule_kind,
405422
schedule_params=schedule_params,
423+
device="B70" if schedule_kind == "mlp" else None,
406424
stop_at_stage=stop_at_stage,
407425
benchmark=benchmark,
408426
payload_func_name=payload_func_name,
@@ -421,7 +439,8 @@ def infer_params_and_lower(
421439
def lower_to_llvm(
422440
mod: ir.Module,
423441
schedule_kind: str,
424-
schedule_params: list[dict],
442+
schedule_params: ScheduleParameters,
443+
device: str | None,
425444
stop_at_stage: str | None,
426445
benchmark: bool,
427446
payload_func_name: str,
@@ -432,6 +451,7 @@ def lower_to_llvm(
432451
schedule = mlp_schedule(
433452
params=schedule_params,
434453
payload_func_name=payload_func_name,
454+
device=device,
435455
stop_at_stage=stop_at_stage,
436456
)
437457
elif schedule_kind == "elemwise":
@@ -441,9 +461,8 @@ def lower_to_llvm(
441461
stop_at_stage=stop_at_stage,
442462
)
443463
elif schedule_kind == "reduction":
444-
layer_params = schedule_params[0]
445464
schedule = reduction_schedule(
446-
parameters=layer_params,
465+
params=schedule_params,
447466
payload_func_name=payload_func_name,
448467
stop_at_stage=stop_at_stage,
449468
)
@@ -495,10 +514,7 @@ def lower_and_execute_benchmark(
495514
A dictionary containing benchmark performance metrics.
496515
"""
497516
assert datatype in ["f16", "bf16"], "Unsupported datatype"
498-
model_dtype = {
499-
"f16": torch.float16,
500-
"bf16": torch.bfloat16,
501-
}[datatype]
517+
model_dtype = dtype_to_torch_dtype(datatype)
502518
execute = stop_at_stage is None
503519

504520
# import torch model
@@ -527,7 +543,6 @@ def lower_and_execute_benchmark(
527543
with torch.no_grad():
528544
torch_model = torch_model.to("xpu")
529545
result_ref = torch_model(*torch_inputs).to("cpu")
530-
torch.xpu.synchronize()
531546

532547
# compile and execute the model with the MLIR backend
533548
torch_all_inputs = [*torch_model.parameters(), *torch_inputs]

‎examples/xegpu/layer_norm.py‎

Lines changed: 17 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
from lighthouse.ingress.mlir_gen.gpu_layer_norm_payload import (
2121
generate_gpu_layer_norm_payload,
2222
)
23+
from lighthouse.schedule.parameters import ScheduleParameters
2324
from lighthouse.schedule.xegpu import reduction_schedule, xegpu_to_binary
2425

2526

@@ -134,7 +135,9 @@ def payload_module(self) -> ir.Module:
134135
return mod
135136

136137
def schedule_modules(
137-
self, stop_at_stage: str | None = None, parameters: dict | None = None
138+
self,
139+
stop_at_stage: str | None = None,
140+
parameters: ScheduleParameters | None = None,
138141
) -> list[ir.Module]:
139142
"""Generate transform schedule for layer_norm."""
140143
schedules = []
@@ -144,7 +147,7 @@ def schedule_modules(
144147
reduction_schedule(
145148
payload_func_name=self.payload_function_name,
146149
stop_at_stage=stop_at_stage,
147-
parameters=parameters,
150+
params=parameters,
148151
)
149152
)
150153

@@ -253,13 +256,18 @@ def parse_cli():
253256
if __name__ == "__main__":
254257
args = parse_cli()
255258

256-
params = {
257-
"sizes": args.sizes,
258-
"wg_rows": args.wg_rows,
259-
"sg_rows": args.sg_rows,
260-
"subgroup_size": args.subgroup_size,
261-
"reduction_step_size": args.reduction_step_size,
262-
}
259+
params = ScheduleParameters(
260+
[
261+
{
262+
"layer_kind": "reduction",
263+
"sizes": args.sizes,
264+
"wg_rows": args.wg_rows,
265+
"sg_rows": args.sg_rows,
266+
"subgroup_size": args.subgroup_size,
267+
"reduction_step_size": args.reduction_step_size,
268+
}
269+
]
270+
)
263271

264272
M, N = args.sizes
265273
dtype = "f32"

0 commit comments

Comments
 (0)