Skip to content

Commit e207770

Browse files
authored
[dependencies] Install torch-mlir from PyPI (#302)
- torch-mlir is now installed from PyPI. - Bumps torch-mlir version to 20261001. - Bumps mlir-python-bindings version. - Imported IR is now slightly different for some kernels, various fixes in lowering schedules. Addresses #283. Assisted-by: Claude, GPT
1 parent da54528 commit e207770

32 files changed

Lines changed: 4305 additions & 308 deletions

‎examples/llama/ref_model.py‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -273,9 +273,13 @@ def forward(self, tokens: torch.Tensor, start_pos: int):
273273
# only for the new sequence. Thus, the matrix of scores is of size
274274
# (seqlen, cache_len + seqlen), and the only masked entries are (i, j) for
275275
# j > cache_len + i, since row i corresponds to token cache_len + i.
276-
mask = torch.hstack(
277-
[torch.zeros((seqlen, start_pos), device=tokens.device), mask]
278-
).type_as(h)
276+
# Skip the empty (seqlen, 0) prefix when start_pos == 0: torch-mlir
277+
# cannot lower the resulting zero-sized aten.empty_strided.
278+
if start_pos > 0:
279+
mask = torch.hstack(
280+
[torch.zeros((seqlen, start_pos), device=tokens.device), mask]
281+
)
282+
mask = mask.type_as(h)
279283

280284
for layer in self.layers:
281285
h = layer(h, start_pos, freqs_cis, mask)

‎examples/xegpu/kernel_bench.py‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@
5151

5252
import torch
5353
import torch._dynamo as dynamo
54+
from torch.nn.attention import sdpa_kernel, SDPBackend
5455
import os
5556
import gc
5657
import numpy as np
@@ -284,7 +285,7 @@ def infer_parameters(
284285
"sg_rows": 16,
285286
"subgroup_size": 16,
286287
"reduction_tile": 64,
287-
"q_load_tile": [16, 32],
288+
"q_load_tile": [16, 16],
288289
"v_load_tile": [32, 32],
289290
"prefetch_tile": [16, 32],
290291
"nb_prefetch": 1,
@@ -727,7 +728,11 @@ def lower_and_execute_benchmark(
727728
# BackendCompilerFailed) and we stop here; in execute mode it runs the
728729
# compiled kernel on the device.
729730
try:
730-
with torch.no_grad():
731+
# Force the math SDPA backend so attention is decomposed into
732+
# supportable ops; on XPU the dispatcher otherwise selects the fused
733+
# '_scaled_dot_product_fused_attention_overrideable' op that fails to
734+
# legalize.
735+
with torch.no_grad(), sdpa_kernel(SDPBackend.MATH):
731736
result = torch_model(*torch_inputs)
732737
except TypeError as e:
733738
msg = str(e)

‎lighthouse/dialects/transform/transform_ext/__init__.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
from .ops.param_cmp_eq import param_cmp_eq
1010
from .ops.replace import replace
1111
from .ops.convert_func_results_to_args import convert_func_results_to_args
12+
from .ops.narrow_vector_contract_operands import narrow_vector_contract_operands
1213
from .ops.emit_definite_failure import emit_definite_failure
1314
from .ops.extract_handle import extract_handle
1415
from .ops.get_tileable_consumers import get_tileable_consumers
@@ -22,10 +23,13 @@
2223
from .ops.filter_num_loops import filter_num_loops
2324
from .ops.filter_elementwise import filter_elementwise
2425
from .ops.fuse_elementwise_op import fuse_elementwise_op
26+
from .ops.fold_reshape_ops_by_expansion import fold_reshape_ops_by_expansion
27+
from .ops.fold_scalar_or_splat_constant import fold_scalar_or_splat_constant
2528
from .ops.filter_by_name import filter_by_name
2629
from .ops.filter_reduction_ops import filter_reduction_ops
2730
from .ops.fuse_dependent_reduction_ops import fuse_dependent_reduction_ops
2831
from .ops.filter_non_contraction_reductions import filter_non_contraction_reductions
32+
from .ops.rewrite_attention_row_mask import rewrite_attention_row_mask
2933
from .ops.filter_contraction_ops import filter_contraction_ops
3034
from .ops.get_leading_unit_tile_sizes import get_leading_unit_tile_sizes
3135
from .ops.move_offsets_to_subview import move_offsets_to_subview
@@ -57,6 +61,8 @@
5761
"filter_non_contraction_reductions",
5862
"filter_num_loops",
5963
"filter_reduction_ops",
64+
"fold_reshape_ops_by_expansion",
65+
"fold_scalar_or_splat_constant",
6066
"fuse_dependent_reduction_ops",
6167
"fuse_elementwise_op",
6268
"get_fusion_roots",
@@ -70,11 +76,13 @@
7076
"infer_xegpu_gemm_params",
7177
"infer_xegpu_reduction_params",
7278
"move_offsets_to_subview",
79+
"narrow_vector_contract_operands",
7380
"param_cmp_eq",
7481
"propagate_tile_sizes",
7582
"register_and_load",
7683
"replace",
7784
"reverse_handles",
85+
"rewrite_attention_row_mask",
7886
"sfc_remap_forall",
7987
"sink_extract_slice_into_loop",
8088
"sink_normalization_past_contraction",
Lines changed: 243 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,243 @@
1+
"""Temporary static-tensor subset of MLIR's reshape-by-expansion patterns."""
2+
3+
from math import prod
4+
5+
from mlir import ir
6+
from mlir.dialects import arith, ext, linalg, tensor, transform
7+
from mlir.dialects.transform import DiagnosedSilenceableFailure
8+
9+
from lighthouse.dialects.transform.transform_ext import TransformExtensionDialect
10+
from lighthouse.utils.mlir import indexing_maps, opview
11+
12+
13+
def _producer(value):
14+
return opview(value.owner) if isinstance(value, ir.OpResult) else None
15+
16+
17+
def _groups(reshape):
18+
return [[int(dim) for dim in group] for group in reshape.reassociation]
19+
20+
21+
def _plan(generic, operand_number, reshape):
22+
maps = indexing_maps(generic)
23+
if (
24+
not generic.outputs
25+
or not all(isinstance(v.type, ir.RankedTensorType) for v in generic.outputs)
26+
or any(not m.is_projected_permutation or m.n_symbols for m in maps)
27+
or not maps[operand_number].results
28+
or any(op.regions for op in generic.regions[0].blocks[0].operations)
29+
):
30+
return None
31+
32+
extents = [None] * len(generic.iterator_types)
33+
for value, m in zip(generic.operands, maps):
34+
ty = value.type
35+
if not isinstance(ty, ir.RankedTensorType):
36+
if isinstance(ty, ir.ShapedType) or m.results:
37+
return None
38+
continue
39+
if ty.encoding is not None or any(size <= 0 for size in ty.shape):
40+
return None
41+
for expr, size in zip(m.results, ty.shape):
42+
dim = expr.position
43+
if extents[dim] is not None and extents[dim] != size:
44+
return None
45+
extents[dim] = size
46+
if any(size is None for size in extents):
47+
return None
48+
49+
expanded_type = (
50+
reshape.src.type
51+
if isinstance(reshape, tensor.CollapseShapeOp)
52+
else reshape.result.type
53+
)
54+
if expanded_type.encoding is not None or any(s <= 0 for s in expanded_type.shape):
55+
return None
56+
shapes = [[size] for size in extents]
57+
for expr, group in zip(maps[operand_number].results, _groups(reshape)):
58+
shape = [expanded_type.shape[i] for i in group]
59+
if prod(shape) != extents[expr.position]:
60+
return None
61+
shapes[expr.position] = shape
62+
if all(len(shape) == 1 for shape in shapes):
63+
return None
64+
65+
loop_groups = []
66+
num_loops = 0
67+
for shape in shapes:
68+
loop_groups.append(list(range(num_loops, num_loops + len(shape))))
69+
num_loops += len(shape)
70+
return maps, shapes, loop_groups, num_loops
71+
72+
73+
def _expand(value, shape, groups):
74+
ty = value.type
75+
if not isinstance(ty, ir.RankedTensorType) or list(ty.shape) == shape:
76+
return value
77+
expanded_type = ir.RankedTensorType.get(shape, ty.element_type)
78+
producer = _producer(value)
79+
if (
80+
isinstance(producer, tensor.CollapseShapeOp)
81+
and producer.src.type == expanded_type
82+
and _groups(producer) == groups
83+
):
84+
return producer.src
85+
return tensor.ExpandShapeOp(expanded_type, value, groups, [], shape).result
86+
87+
88+
def _rewrite(generic, reshape, plan, rewriter):
89+
maps, shapes, loop_groups, num_loops = plan
90+
with generic.location, ir.InsertionPoint(generic):
91+
operands, new_maps, reassociations = [], [], []
92+
for value, m in zip(generic.operands, maps):
93+
shape, groups, exprs = [], [], []
94+
for expr in m.results:
95+
sizes = shapes[expr.position]
96+
groups.append(list(range(len(shape), len(shape) + len(sizes))))
97+
shape.extend(sizes)
98+
exprs.extend(
99+
ir.AffineExpr.get_dim(d) for d in loop_groups[expr.position]
100+
)
101+
operands.append(_expand(value, shape, groups))
102+
new_maps.append(ir.AffineMapAttr.get(ir.AffineMap.get(num_loops, 0, exprs)))
103+
reassociations.append(groups)
104+
105+
expanded = opview(generic.operation.clone())
106+
for i, value in enumerate(operands):
107+
expanded.operands[i] = value
108+
expanded.attributes["indexing_maps"] = ir.ArrayAttr.get(new_maps)
109+
expanded.attributes["iterator_types"] = ir.ArrayAttr.get(
110+
[it for it, shape in zip(generic.iterator_types, shapes) for _ in shape]
111+
)
112+
for result, output in zip(expanded.results, expanded.outputs):
113+
result.set_type(output.type)
114+
115+
# An original index denotes the linearized coordinate, not just the
116+
# first of the expanded loop dimensions.
117+
for index in list(expanded.regions[0].blocks[0].operations):
118+
if not isinstance(index, linalg.IndexOp):
119+
continue
120+
dim = int(index.dim)
121+
with ir.InsertionPoint(index):
122+
coord = linalg.IndexOp(loop_groups[dim][0]).result
123+
for new_dim, size in zip(loop_groups[dim][1:], shapes[dim][1:]):
124+
stride = arith.ConstantOp(ir.IndexType.get(), size)
125+
coord = arith.AddIOp(
126+
arith.MulIOp(coord, stride.result).result,
127+
linalg.IndexOp(new_dim).result,
128+
).result
129+
rewriter.replace_op(index, [coord])
130+
131+
replacements = []
132+
for i, (old, new) in enumerate(zip(generic.results, expanded.results)):
133+
if old.type == new.type:
134+
replacements.append(new)
135+
else:
136+
replacements.append(
137+
tensor.CollapseShapeOp(
138+
old.type, new, reassociations[len(generic.inputs) + i]
139+
).result
140+
)
141+
142+
if isinstance(reshape, tensor.ExpandShapeOp):
143+
result_number = reshape.src.result_number
144+
rewriter.replace_op(reshape, [expanded.results[result_number]])
145+
rewriter.replace_op(generic, replacements)
146+
147+
148+
def _fold_one(root, rewriter):
149+
candidates = []
150+
151+
def visit(operation):
152+
if operation.name in ("linalg.generic", "tensor.expand_shape"):
153+
candidates.append(operation.opview)
154+
return ir.WalkResult.ADVANCE
155+
156+
root.walk(visit)
157+
for candidate in candidates:
158+
if isinstance(candidate, linalg.GenericOp):
159+
for i, value in enumerate(candidate.inputs):
160+
reshape = _producer(value)
161+
if not isinstance(reshape, tensor.CollapseShapeOp):
162+
continue
163+
plan = _plan(candidate, i, reshape)
164+
if plan is not None:
165+
_rewrite(candidate, reshape, plan, rewriter)
166+
return True
167+
else:
168+
generic = _producer(candidate.src)
169+
# Do not rewrite a producer outside the requested container.
170+
if not isinstance(generic, linalg.GenericOp) or generic not in candidates:
171+
continue
172+
i = len(generic.inputs) + candidate.src.result_number
173+
plan = _plan(generic, i, candidate)
174+
if plan is not None:
175+
_rewrite(generic, candidate, plan, rewriter)
176+
return True
177+
return False
178+
179+
180+
class FoldReshapeOpsByExpansionOp(
181+
TransformExtensionDialect.Operation, name="fold_reshape_ops_by_expansion"
182+
):
183+
"""Fold tensor reshapes into generic loop dimensions without body fusion.
184+
185+
Temporary replacement for the not-yet-exposed native pattern collection.
186+
Within one func.func or builtin.module, greedily fold collapse_shape inputs
187+
and expand_shape results of linalg.generic. All legal matches are accepted,
188+
including shared producers; collapsed results preserve other users.
189+
190+
Supports positive static, unencoded tensors, projected-permutation indexing
191+
maps, scalar inputs and region-free bodies (including linalg.index).
192+
Dynamic/encoded tensors, non-projected maps, named ops, nested scalar regions
193+
and the native pad patterns are not implemented and are left unchanged.
194+
Run canonicalization afterwards to clean up introduced reshapes.
195+
196+
Consumes the container handle and invalidates nested handles. Reacquire
197+
payload handles through the returned container after this transform.
198+
"""
199+
200+
target: ext.Operand[transform.AnyOpType]
201+
result: ext.Result[transform.AnyOpType[()]] = ext.infer_result()
202+
203+
@classmethod
204+
def attach_interface_impls(cls, context=None):
205+
cls.TransformOpInterfaceModel.attach(cls.OPERATION_NAME, context=context)
206+
cls.MemoryEffectsOpInterfaceModel.attach(cls.OPERATION_NAME, context=context)
207+
208+
class TransformOpInterfaceModel(transform.TransformOpInterface):
209+
@staticmethod
210+
def apply(op, rewriter, results, state) -> DiagnosedSilenceableFailure:
211+
targets = [target.operation for target in state.get_payload_ops(op.target)]
212+
if len(targets) != 1 or targets[0].name not in (
213+
"func.func",
214+
"builtin.module",
215+
):
216+
op.operation.location.emit_error(
217+
"fold_reshape_ops_by_expansion expects one func.func or builtin.module"
218+
)
219+
return DiagnosedSilenceableFailure.SilenceableFailure
220+
while _fold_one(targets[0], rewriter):
221+
pass
222+
results.set_ops(op.result, targets)
223+
return DiagnosedSilenceableFailure.Success
224+
225+
@staticmethod
226+
def allow_repeated_handle_operands(_op) -> bool:
227+
return False
228+
229+
class MemoryEffectsOpInterfaceModel(ir.MemoryEffectsOpInterface):
230+
@staticmethod
231+
def get_effects(op):
232+
return (
233+
transform.consumes_handle(op.op_operands)
234+
+ transform.produces_handle(op.results)
235+
+ transform.modifies_payload()
236+
)
237+
238+
239+
def fold_reshape_ops_by_expansion(
240+
target: ir.Value[transform.AnyOpType],
241+
) -> ir.Value[transform.AnyOpType]:
242+
"""Expand generic iteration spaces across static tensor reshape boundaries."""
243+
return FoldReshapeOpsByExpansionOp(target=target).result

0 commit comments

Comments
 (0)