Skip to content

Commit 8cd3827

Browse files
committed
clean up
1 parent ce1725d commit 8cd3827

2 files changed

Lines changed: 26 additions & 40 deletions

File tree

‎examples/xegpu/fused_attention.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,10 @@ def check_correctness(
107107

108108
class XeGPUFusedAttention:
109109
"""
110-
Fused attention workload on XeGPU.
110+
Fused attention workload on XeGPU. This workload starts with standard attention
111+
at linalg level and applies a series of transformations to arrive at a fused
112+
attention kernel where each work group computes a tile of the output with the
113+
fused attention algorithm.
111114
112115
Computes fused attention:
113116
output = softmax(Q @ K^T / sqrt(n_head)) @ V

‎lighthouse/schedule/xegpu/fused_attention_schedule.py‎

Lines changed: 22 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@
2222
from lighthouse.schedule import schedule_boilerplate
2323
from lighthouse.dialects.transform.transform_ext import (
2424
replace_with_fused_attention,
25-
update_address_space,
2625
)
2726

2827

@@ -31,14 +30,15 @@ def fused_attention_schedule(
3130
parameters: Optional[dict] = None,
3231
) -> ir.Module:
3332
"""
34-
Generate transform schedule for fused attention operation.
33+
Generate transform schedule for attention kernel.
3534
3635
The schedule performs the following transformations:
37-
1. Tile the fused attention operation
36+
1. Tile the fuse the strandard attention computation along parallel dims
3837
2. Vectorize operations
3938
3. Bufferize tensors
40-
4. Convert to GPU dialect
41-
5. Lower to XeGPU operations
39+
4. Perform the fused attention optimization for the innermost computation
40+
5. Convert to GPU dialect
41+
6. Lower to XeGPU operations
4242
4343
Args:
4444
stop_at_stage: Optional stage name to stop early (for debugging)
@@ -86,7 +86,7 @@ def bundle_xegpu_fused_attention_schedule(
8686
parameters: dict,
8787
stop_at_stage: str = "",
8888
) -> ir.Value[transform.AnyOpType]:
89-
"""Schedule for lowering fused attention payload to xegpu wg level."""
89+
"""Schedule for lowering attention payload to xegpu wg level."""
9090

9191
if stop_at_stage == "initial":
9292
raise PipelineInterrupt()
@@ -165,6 +165,7 @@ def bundle_xegpu_fused_attention_schedule(
165165
transform.apply_cse(func)
166166
canonicalize(func)
167167

168+
# Fuse the remaining operations into the scf.forall loop.
168169
linalg_mul_op = match_and_split(func, ops={"linalg.mul"}, nhandles=1)[0]
169170
first_matmul = transform.get_producer_of_operand(
170171
anytype, linalg_mul_op, operand_number=0
@@ -197,7 +198,7 @@ def bundle_xegpu_fused_attention_schedule(
197198
if stop_at_stage == "outer-tiled":
198199
raise PipelineInterrupt()
199200

200-
# vectorize
201+
# Vectorize
201202
func = structured.VectorizeChildrenAndApplyPatternsOp(
202203
func,
203204
fold_type_extensions_into_contract=True,
@@ -212,7 +213,7 @@ def bundle_xegpu_fused_attention_schedule(
212213
if stop_at_stage == "vectorized":
213214
raise PipelineInterrupt()
214215

215-
# bufferize
216+
# Bufferize
216217
mod = apply_registered_pass(mod, "eliminate-empty-tensors")
217218
identity_layout = LayoutMapOption.IdentityLayoutMap
218219
mod = transform_bufferization.OneShotBufferizeOp(
@@ -221,23 +222,13 @@ def bundle_xegpu_fused_attention_schedule(
221222
bufferize_function_boundaries=True,
222223
function_boundary_type_conversion=identity_layout,
223224
).result
224-
transform.apply_cse(mod)
225-
canonicalize(mod)
226225
# fold memref.subviews into vector.transfer_read/write ops
227226
mod = apply_registered_pass(mod, "fold-memref-alias-ops")
228227
transform.apply_cse(mod)
229228
canonicalize(mod)
230229

231-
# promote memref.alloc to memref.alloca in payload function
232-
func = match(mod, ops={"func.func"})
233-
func = apply_registered_pass(
234-
func,
235-
"promote-buffers-to-stack",
236-
options={
237-
"max-alloc-size-in-bytes": "8192",
238-
"max-rank-of-allocated-memref": "2",
239-
},
240-
)
230+
if stop_at_stage == "bufferized":
231+
raise PipelineInterrupt()
241232

242233
# Extract q, k, v memrefs from the bufferized IR
243234
# Match vector.contract ops to find the q, k, v loads
@@ -256,8 +247,8 @@ def bundle_xegpu_fused_attention_schedule(
256247
anytype, first_contract, operand_number=1
257248
)
258249

259-
# # Second vector.contract is attention_weights @ V
260-
# # Its second operand is the v load (vector.transfer_read)
250+
# Second vector.contract is attention_weights @ V
251+
# Its second operand is the v load (vector.transfer_read)
261252
second_contract = contract_ops[1]
262253
v_load = transform.get_producer_of_operand(
263254
anytype, second_contract, operand_number=1
@@ -268,12 +259,9 @@ def bundle_xegpu_fused_attention_schedule(
268259
mulf_op = match_and_split(func, ops={"arith.mulf"}, nhandles=1)[0]
269260
scale = transform.get_producer_of_operand(anytype, mulf_op, operand_number=1)
270261

271-
if stop_at_stage == "bufferized":
272-
raise PipelineInterrupt()
273-
274-
# Generate fused attention computation with inner tiling
275-
# This replaces the second vector.contract (attention_weights @ V) with a tiled
276-
# loop that implements online softmax for efficient memory usage
262+
# Apply the fused attention optimization. This replaces the second vector.contract
263+
# (attention_weights @ V) with a tiled loop that implements online softmax for
264+
# efficient memory usage
277265
tile_size = parameters.get(
278266
"inner_loop_tile_size", 64
279267
) # Tile size for reduction dimension (K/V sequence length)
@@ -291,20 +279,20 @@ def bundle_xegpu_fused_attention_schedule(
291279
if stop_at_stage == "inner-tiled":
292280
raise PipelineInterrupt()
293281

294-
# convert forall to parallel
282+
# Convert forall to parallel
295283
wg_loops = match_and_split(mod, ops={"scf.forall"})
296284
for wg_loop in wg_loops:
297285
wg_loop = loop.loop_forall_to_parallel([anytype], wg_loop)
298286
func = transform.get_parent_op(anytype, wg_loop)
299287

300-
# convert scf.parallel to gpu.launch
288+
# Convert scf.parallel to gpu.launch
301289
func = apply_registered_pass(func, "gpu-map-parallel-loops")
302290
func = apply_registered_pass(func, "convert-parallel-loops-to-gpu")
303291
func = apply_registered_pass(func, "lower-affine")
304292
transform.apply_cse(func)
305293
canonicalize(func)
306294

307-
# set the number of threads for the gpu.launch operation
295+
# Set the number of threads for the gpu.launch operation
308296
launch_op = match_and_split(func, ops={"gpu.launch"})
309297
wg_rows = parameters["wg_rows"]
310298
sg_rows = parameters["sg_rows"]
@@ -313,7 +301,7 @@ def bundle_xegpu_fused_attention_schedule(
313301
num_threads = num_subgroups * subgroup_size
314302
xegpu.set_gpu_launch_threads(launch_op[0], threads=[num_threads, 1, 1])
315303

316-
# outline gpu func
304+
# Outline gpu func
317305
func = apply_registered_pass(func, "lower-affine")
318306
canonicalize(func)
319307
func = apply_registered_pass(func, "gpu-launch-sink-index-computations")
@@ -323,22 +311,17 @@ def bundle_xegpu_fused_attention_schedule(
323311
if stop_at_stage == "gpu-outlining":
324312
raise PipelineInterrupt()
325313

326-
# set xevm target
314+
# Set xevm target
327315
mod = apply_registered_pass(
328316
mod,
329317
"xevm-attach-target",
330318
options={"O": "3", "chip": "bmg"},
331319
)
332320

333-
# for each gpu function in the gpu module, change memref.alloca address
334-
# space to 3 (SLM) and convert vector to xegpu.
321+
# Convert vectot to xegpu
335322
gpu_mod_ops = match_and_split(mod, ops={"gpu.module"})
336323
for gpu_mod in gpu_mod_ops:
337324
gpu_func = match(gpu_mod, ops={"gpu.func"})
338-
allocas = match_and_split(gpu_func, ops={"memref.alloca"}, nhandles=3)
339-
for alloca in allocas:
340-
# print("Updating address space for alloca:")
341-
update_address_space(alloca, address_space=3)
342325
gpu_func = apply_registered_pass(gpu_func, "convert-vector-to-xegpu")
343326
transform.apply_cse(gpu_func)
344327
gpu_func = apply_registered_pass(gpu_func, "loop-invariant-code-motion")

0 commit comments

Comments
 (0)