Skip to content

Commit d34fc8a

Browse files
authored
[xegpu] Generalize fused attention schedule for KernelBench (#272)
- Generalizes op matching in `fused_attention_schedule` so that it also applies to KernelBench level1 97 ScaledAttention kernel. - Fixes duplicated handle issue in `trace_producers` trasform ext op. - Adds a new `filter_contraction_ops` transform ext op to match batch_matmul with optionally fused elementwise ops.
1 parent 33af27c commit d34fc8a

5 files changed

Lines changed: 234 additions & 16 deletions

File tree

‎lighthouse/dialects/transform/transform_ext/__init__.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
from .ops.filter_elementwise import filter_elementwise
2020
from .ops.filter_by_name import filter_by_name
2121
from .ops.filter_reduction_ops import filter_reduction_ops
22+
from .ops.filter_contraction_ops import filter_contraction_ops
2223
from .ops.get_leading_unit_tile_sizes import get_leading_unit_tile_sizes
2324
from .ops.move_offsets_to_subview import move_offsets_to_subview
2425
from .ops.clear_tile_and_fuse_annotations import clear_tile_and_fuse_annotations
@@ -33,6 +34,7 @@
3334
"convert_func_results_to_args",
3435
"extract_handle",
3536
"filter_by_name",
37+
"filter_contraction_ops",
3638
"filter_elementwise",
3739
"filter_num_loops",
3840
"filter_reduction_ops",
Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
from mlir import ir
2+
from mlir.dialects import transform, linalg
3+
4+
from lighthouse.dialects.transform.transform_ext.utils.make_filter_handles_op import (
5+
make_filter_handles_op,
6+
)
7+
from lighthouse.utils.mlir import opview, indexing_maps, linalg_inputs, dim_position
8+
9+
10+
def _map_dims(m: ir.AffineMap) -> set[int]:
11+
"""Set of iteration-dim positions referenced by an affine map's results."""
12+
dims = set()
13+
for r in m.results:
14+
pos = dim_position(r)
15+
if pos is not None:
16+
dims.add(pos)
17+
return dims
18+
19+
20+
def _is_structural_contraction(ov: ir.OpView) -> bool:
21+
"""Detect a matmul-like op from its indexing maps alone (body-agnostic).
22+
23+
A contraction has a reduction dimension (one that is absent from the
24+
output) that is shared by at least two inputs. This ignores the body, so it
25+
also matches contractions with extra elementwise operands (e.g. a scaled or
26+
dequantized input) that the strict multiply-accumulate matcher rejects.
27+
"""
28+
maps = indexing_maps(ov)
29+
inputs = linalg_inputs(ov)
30+
if not maps or not inputs:
31+
return False
32+
num_inputs = len(inputs)
33+
input_maps = maps[:num_inputs]
34+
output_maps = maps[num_inputs:]
35+
36+
out_dims: set[int] = set()
37+
for m in output_maps:
38+
out_dims |= _map_dims(m)
39+
40+
# Reduction dims are the iteration dims not present in any output.
41+
reduction_dims = set(range(maps[0].n_dims)) - out_dims
42+
for k in reduction_dims:
43+
if sum(1 for m in input_maps if k in _map_dims(m)) >= 2:
44+
return True
45+
return False
46+
47+
48+
def is_contraction_op(op: ir.Operation | ir.OpView) -> bool:
49+
"""Check whether the op is a linalg contraction (matmul-like) op.
50+
51+
Recognizes both named contractions (e.g. linalg.batch_matmul) and their
52+
generic form, independent of rank. A contraction contracts a reduction
53+
dimension shared by two inputs, which distinguishes it from the surrounding
54+
elementwise or plain (single-input) reduction ops.
55+
"""
56+
ov = opview(op)
57+
if "linalg" not in ov.operation.name:
58+
return False
59+
return linalg.isa_contraction_op(ov) or _is_structural_contraction(ov)
60+
61+
62+
FilterContractionOpsOp = make_filter_handles_op(
63+
"filter_contraction_ops", is_contraction_op
64+
)
65+
66+
67+
def filter_contraction_ops(target: ir.Value[transform.AnyOpType]) -> ir.Value:
68+
"""
69+
snake_case wrapper to create a FilterContractionOpsOp.
70+
71+
Args:
72+
target: Handle to target op(s).
73+
Returns:
74+
Handle to the contraction-op subset of `target`.
75+
"""
76+
return FilterContractionOpsOp(target=target).ops

‎lighthouse/dialects/transform/transform_ext/ops/trace_producers.py‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -41,13 +41,13 @@ def apply(
4141
# Walk the SSA producer graph via operand -> owner edges.
4242
# Use BFS to guarantee closest-first ordering by graph distance.
4343
producers: list[ir.Operation] = []
44-
visited_ids: set[int] = set()
44+
visited: set[ir.Operation] = set()
4545
worklist = deque()
4646

4747
for operand in leaf.operands:
4848
owner_op = defining_op(operand)
49-
if owner_op is not None and id(owner_op) not in visited_ids:
50-
visited_ids.add(id(owner_op))
49+
if owner_op is not None and owner_op not in visited:
50+
visited.add(owner_op)
5151
worklist.append(owner_op)
5252

5353
while worklist:
@@ -56,8 +56,8 @@ def apply(
5656

5757
for operand in producer.operands:
5858
owner_op = defining_op(operand)
59-
if owner_op is not None and id(owner_op) not in visited_ids:
60-
visited_ids.add(id(owner_op))
59+
if owner_op is not None and owner_op not in visited:
60+
visited.add(owner_op)
6161
worklist.append(owner_op)
6262

6363
results.set_ops(op.ops, producers)

‎lighthouse/schedule/xegpu/fused_attention_schedule.py‎

Lines changed: 49 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -151,6 +151,13 @@ def bundle_xegpu_fused_attention_schedule(
151151
# Match linalg.softmax operation if any and decompose it into generic ops
152152
softmax_ops = structured.structured_match(anytype, func, ops=["linalg.softmax"])
153153
structured.structured_decompose_interface(anytype, softmax_ops)
154+
# Convert linalg.mul and linalg.batch_matmul to linalg.generic
155+
structured.structured_generalize(
156+
anytype,
157+
structured.structured_match(
158+
anytype, func, ops=["linalg.mul", "linalg.batch_matmul"]
159+
),
160+
)
154161

155162
# Normalize possible singleton dimensions so tile+fuse logic works.
156163
with ir.InsertionPoint(transform.apply_patterns(func).patterns):
@@ -197,20 +204,51 @@ def bundle_xegpu_fused_attention_schedule(
197204
# Q@K^T: linalg.batch_matmul(q_slice, linalg.transpose(k_slice))
198205
# scale: linalg.mul(qkt, linalg.fill(scale_constant))
199206
# P@V: linalg.batch_matmul(softmax_out, v_slice)
200-
matmul_ops = match_and_split(func, ops={"linalg.batch_matmul"}, nhandles=2)
207+
208+
linalg_ops = structured.structured_match(
209+
anytype, func, ops=["linalg.generic", "linalg.batch_matmul"]
210+
)
211+
contraction_ops = transform_ext.filter_contraction_ops(linalg_ops)
212+
213+
# Match max reduction op. Assumes there's only one arith.max* op.
214+
arith_max_op = match_and_split(
215+
func, ops=["arith.maximumf", "arith.maxnumf"], nhandles=1
216+
)[0]
217+
max_reduction = transform.get_parent_op(
218+
anytype, arith_max_op, op_name="linalg.generic"
219+
)
220+
221+
def get_producers_by_name(target, op_names):
222+
producers = transform_ext.trace_producers(target)
223+
return transform_ext.filter_by_name(producers, op_names=op_names)
224+
225+
# Trace the scalar from max reduction producer chain.
226+
max_producer_generics = get_producers_by_name(
227+
max_reduction, op_names="linalg.generic"
228+
)
229+
# Assume the scaling happens in the first ancestor.
230+
max_producer = transform_ext.extract_handle(max_producer_generics, 0)
231+
max_scale_mul_op = match_and_split(max_producer, ops={"arith.mulf"}, nhandles=1)[0]
232+
scale_producers = transform_ext.trace_producers(max_scale_mul_op)
233+
scale_const_op = transform_ext.extract_handle(
234+
transform_ext.filter_by_name(scale_producers, op_names="arith.constant"), 0
235+
)
236+
237+
matmul_ops = transform.split_handle(2 * [anytype], contraction_ops)
201238
qk_matmul, pv_matmul = matmul_ops[0], matmul_ops[1]
202239

203-
q = transform.get_producer_of_operand(anytype, qk_matmul, operand_number=0)
204-
k_transpose = transform.get_producer_of_operand(
205-
anytype, qk_matmul, operand_number=1
240+
# Find the tensor.extract_slice producers for the Q@K^T matmul.
241+
qk_extract_slice_producers = get_producers_by_name(
242+
qk_matmul, op_names="tensor.extract_slice"
206243
)
207-
k = transform.get_producer_of_operand(anytype, k_transpose, operand_number=0)
208-
v = transform.get_producer_of_operand(anytype, pv_matmul, operand_number=1)
244+
q = transform_ext.extract_handle(qk_extract_slice_producers, 0)
245+
k = transform_ext.extract_handle(qk_extract_slice_producers, 1)
209246

210-
# The scale is the fill value of the linalg.mul rhs operand.
211-
mul_op = match_and_split(func, ops={"linalg.mul"}, nhandles=1)[0]
212-
scale_fill = transform.get_producer_of_operand(anytype, mul_op, operand_number=1)
213-
scale = transform.get_producer_of_operand(anytype, scale_fill, operand_number=0)
247+
# Find handle to v as the first tensor.extract_slice producer of PV matmul
248+
pv_extract_slice_producers = get_producers_by_name(
249+
pv_matmul, op_names="tensor.extract_slice"
250+
)
251+
v = transform_ext.extract_handle(pv_extract_slice_producers, 0)
214252

215253
# Replace the P@V batch matmul with a loop over the K/V sequence length that
216254
# implements online softmax, fusing Q@K^T and the softmax into it.
@@ -221,7 +259,7 @@ def bundle_xegpu_fused_attention_schedule(
221259
q=q,
222260
k=k,
223261
v=v,
224-
scale=scale,
262+
scale=scale_const_op,
225263
output=pv_matmul,
226264
tile_size=reduction_tile,
227265
)
Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
# RUN: %PYTHON %s | FileCheck %s
2+
3+
from mlir import ir
4+
from mlir.dialects import transform
5+
from mlir.dialects.transform import structured
6+
7+
import lighthouse.dialects as lh_dialects
8+
from lighthouse import transform as lh_transform
9+
from lighthouse.dialects.transform import transform_ext
10+
from lighthouse.schedule.builders import schedule_boilerplate
11+
12+
13+
def apply_filter(payload: str, name: str):
14+
with ir.Context(), ir.Location.unknown():
15+
lh_dialects.register_and_load()
16+
module = ir.Module.parse(payload)
17+
with schedule_boilerplate() as (sched, named_seq):
18+
candidates = lh_transform.match_op(
19+
named_seq.bodyTarget, structured.MatchInterfaceEnum.LinalgOp
20+
)
21+
filtered = transform_ext.filter_contraction_ops(candidates)
22+
transform.print_(target=filtered, name=name)
23+
transform.yield_()
24+
sched.body.operations[0].apply(module.operation)
25+
26+
27+
# Named matmul and batch_matmul mixed with elementwise ops.
28+
NAMED = """
29+
#id = affine_map<(d0, d1) -> (d0, d1)>
30+
module {
31+
func.func @main(
32+
%a: tensor<8x8xf32>,
33+
%m0: tensor<4x8xf32>,
34+
%m1: tensor<8x4xf32>,
35+
%x: tensor<2x4x8xf32>,
36+
%y: tensor<2x8x4xf32>)
37+
-> (tensor<8x8xf32>, tensor<4x4xf32>, tensor<2x4x4xf32>) {
38+
%e0 = tensor.empty() : tensor<8x8xf32>
39+
%add = linalg.add ins(%a, %a : tensor<8x8xf32>, tensor<8x8xf32>)
40+
outs(%e0 : tensor<8x8xf32>) -> tensor<8x8xf32>
41+
42+
%e1 = tensor.empty() : tensor<8x8xf32>
43+
%gen = linalg.generic {indexing_maps = [#id, #id], iterator_types = ["parallel", "parallel"]}
44+
ins(%a : tensor<8x8xf32>)
45+
outs(%e1 : tensor<8x8xf32>) {
46+
^bb0(%i: f32, %o: f32):
47+
linalg.yield %i : f32
48+
} -> tensor<8x8xf32>
49+
50+
%e2 = tensor.empty() : tensor<4x4xf32>
51+
%mm = linalg.matmul ins(%m0, %m1 : tensor<4x8xf32>, tensor<8x4xf32>)
52+
outs(%e2 : tensor<4x4xf32>) -> tensor<4x4xf32>
53+
54+
%e3 = tensor.empty() : tensor<2x4x4xf32>
55+
%bm = linalg.batch_matmul ins(%x, %y : tensor<2x4x8xf32>, tensor<2x8x4xf32>)
56+
outs(%e3 : tensor<2x4x4xf32>) -> tensor<2x4x4xf32>
57+
return %gen, %mm, %bm : tensor<8x8xf32>, tensor<4x4xf32>, tensor<2x4x4xf32>
58+
}
59+
}
60+
"""
61+
62+
# CHECK-LABEL: IR printer: NAMED
63+
# CHECK: linalg.matmul
64+
# CHECK: linalg.batch_matmul
65+
# CHECK-NOT: linalg.add
66+
# CHECK-NOT: linalg.generic
67+
apply_filter(NAMED, name="NAMED")
68+
69+
70+
# A linalg.generic with elementwise (extf) ops fused into the matmul body,
71+
# preceded by a linalg.fill that must not be matched.
72+
FUSED = """
73+
#map = affine_map<(d0, d1, d2, d3, d4) -> (d0, d1, d2, d4)>
74+
#map1 = affine_map<(d0, d1, d2, d3, d4) -> (d0, d1, d4, d3)>
75+
#map2 = affine_map<(d0, d1, d2, d3, d4) -> (d0, d1, d2, d3)>
76+
module {
77+
func.func @main(
78+
%a: tensor<1x1x128x1024xbf16>,
79+
%b: tensor<1x1x1024x512xbf16>) -> tensor<1x1x128x512xf32> {
80+
%cst = arith.constant 0.0 : f32
81+
%e = tensor.empty() : tensor<1x1x128x512xf32>
82+
%fill = linalg.fill ins(%cst : f32) outs(%e : tensor<1x1x128x512xf32>) -> tensor<1x1x128x512xf32>
83+
%mm = linalg.generic {indexing_maps = [#map, #map1, #map2],
84+
iterator_types = ["parallel", "parallel", "parallel", "parallel", "reduction"]}
85+
ins(%a, %b : tensor<1x1x128x1024xbf16>, tensor<1x1x1024x512xbf16>)
86+
outs(%fill : tensor<1x1x128x512xf32>) {
87+
^bb0(%in: bf16, %in_18: bf16, %out: f32):
88+
%ea = arith.extf %in : bf16 to f32
89+
%eb = arith.extf %in_18 : bf16 to f32
90+
%mul = arith.mulf %ea, %eb : f32
91+
%acc = arith.addf %out, %mul : f32
92+
linalg.yield %acc : f32
93+
} -> tensor<1x1x128x512xf32>
94+
return %mm : tensor<1x1x128x512xf32>
95+
}
96+
}
97+
"""
98+
99+
# CHECK-LABEL: IR printer: FUSED
100+
# CHECK: linalg.generic
101+
# CHECK-NOT: linalg.fill
102+
apply_filter(FUSED, name="FUSED")

0 commit comments

Comments
 (0)