Skip to content

Commit 2b797af

Browse files
committed
kernel_bench: support rms norm kernel
1 parent 1471ad5 commit 2b797af

1 file changed

Lines changed: 49 additions & 18 deletions

File tree

‎examples/xegpu/kernel_bench.py‎

Lines changed: 49 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -181,6 +181,9 @@ def infer_parameters(
181181
schedule_params = ScheduleParameters([layer_params])
182182
schedule_kind = "elemwise"
183183
elif len(elemwise) > 0 and len(reduction) > 0 and len(batch_matmuls) == 0:
184+
# Elementwise + reduction kernel
185+
iter_space = reduction[0]["iterators"]
186+
n_reduction_dims = sum(s == "reduction" for s in iter_space)
184187
# elemwise + reduction kernel, e.g. softmax or layer norm
185188
shape = elemwise[-1]["shape"]
186189
res_elemtype = elemwise[-1]["elemtype"]
@@ -189,25 +192,46 @@ def infer_parameters(
189192
res_bytes = elemtype_bytes[res_elemtype]
190193
read_bytes = int(np.prod(shape)) * res_bytes
191194
write_bytes = int(np.prod(shape)) * res_bytes
192-
193-
layer_params = {
194-
"layer_kind": "reduction",
195-
"wg_rows": 64,
196-
"sg_rows": 8,
197-
"subgroup_size": 16,
198-
"reduction_step_size": 32,
199-
}
195+
if len(shape) == 2:
196+
# 2d softmax like kernel
197+
layer_params = {
198+
"layer_kind": "reduction",
199+
"wg_tile": [64, 0],
200+
"sg_tile": [8, 0],
201+
"subgroup_size": 16,
202+
"reduction_tile": [0, 32],
203+
}
204+
205+
elif len(shape) == 4 and iter_space[1] == "reduction" and n_reduction_dims == 1:
206+
# 4d rms norm like kernel
207+
layer_params = {
208+
"layer_kind": "reduction",
209+
"sizes": shape,
210+
"wg_tile": [1, 0, 128, 128],
211+
"wg_subtile": [0, 0, 16, 32],
212+
"sg_tile": [0, 2, 16, 32],
213+
"reduction_tile": [0, 32, 0, 0],
214+
"subgroup_size": 16,
215+
}
200216

201217
# Ensure shape is divisible by tile sizes.
202218
# Padding or remainder handling is not implemented yet.
203-
if shape[0] % layer_params["wg_rows"] != 0:
204-
raise ValueError(
205-
f"Shape {shape} dimension 0 not divisible by wg_rows={layer_params['wg_rows']}"
206-
)
207-
if shape[1] % layer_params["reduction_step_size"] != 0:
208-
raise ValueError(
209-
f"Shape {shape} dimension 1 not divisible by reduction_step_size={layer_params['reduction_step_size']}"
210-
)
219+
wg_sizes = (
220+
layer_params["wg_subtile"]
221+
if "wg_subtile" in layer_params
222+
else layer_params["wg_tile"]
223+
)
224+
reduction_tile = layer_params["reduction_tile"]
225+
for i, (size, wg, red) in enumerate(zip(shape, wg_sizes, reduction_tile)):
226+
if wg > 0 and size % wg != 0:
227+
raise ValueError(
228+
f"Shape {shape} dimension {i} not divisible by wg_tile={wg_sizes}"
229+
)
230+
if red > 0 and size % red != 0:
231+
raise ValueError(
232+
f"Shape {shape} dimension {i} not divisible by reduction_tile={reduction_tile}"
233+
)
234+
211235
schedule_params = ScheduleParameters([layer_params])
212236
schedule_kind = "reduction"
213237
elif len(elemwise) > 0 and len(reduction) > 0 and len(batch_matmuls) > 0:
@@ -609,7 +633,7 @@ def lower_and_execute_benchmark(
609633
params_cache_json=f"kb_params_level{level}-{id}.json",
610634
dump_parameters=dump_parameters,
611635
enable_tuning=execute,
612-
verbose=2,
636+
verbose=verbose,
613637
)
614638
backend = gpu_backend(
615639
fn_compile,
@@ -647,13 +671,20 @@ def lower_and_execute_benchmark(
647671
layer.get("kind") == "batch_matmul"
648672
for layer in kernel_metadata.get("layers", [])
649673
)
674+
is_reduction = any(
675+
layer.get("kind") == "reduction"
676+
for layer in kernel_metadata.get("layers", [])
677+
)
650678
atol = abs(result_ref).max() * 1e-3
651679
rtol = 1e-3
652680
if result_ref.dtype == torch.bfloat16:
653681
rtol = 2e-2
654682
if is_attention:
655-
# Attention fuses 2 matmuls and a softmax, needs a looser atol.
683+
# Attention fuses 2 matmuls and a softmax.
656684
atol = 2e-2
685+
elif is_reduction:
686+
# RMS norm currently uses bf16 accumulator.
687+
atol = 6e-2
657688
elif is_mlp:
658689
atol = 7e-3
659690
success = torch.allclose(result, result_ref, rtol=rtol, atol=atol)

0 commit comments

Comments
 (0)