Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
36 commits
Select commit Hold shift + click to select a range
faa5fb4
install torch-mlir from PyPI and pin latest version
tkarna Oct 9, 2026
7158313
sink_normalization_past_contraction: support select mask in op chain
tkarna Oct 6, 2026
d3b074e
fuse_dependent_reduction_ops supports P inf masking
tkarna Oct 13, 2026
a3e9915
transform_ext: add rewrite_attention_row_mask
tkarna Oct 13, 2026
51eed19
fuse_dependent_reduction_ops: compute correction factors only over di…
tkarna Oct 13, 2026
0f6635e
fused_attention_schedule: update wg tiling matchers
tkarna Oct 13, 2026
e5baed6
attention_fusion: update matching and transforms to work with P mask
tkarna Oct 13, 2026
c8d447b
sink_normalization_past_contraction: generalize to handle separate sc…
tkarna Oct 14, 2026
fb36c62
transform_ext: add fold_reshape_ops_by_expansion op
tkarna Oct 14, 2026
ca858d9
fuse_dependent_reduction_ops: do not fuse p mask in to pv matmul
tkarna Oct 14, 2026
00ca284
attention_fusion: drop linalg-fuse-elementwise-ops pass
tkarna Oct 14, 2026
0de9aff
fused_attention_schedule: use fold_reshape_ops_by_expansion ops
tkarna Oct 14, 2026
e288bfa
attention_fusion: fuse P producers into reduction loop
tkarna Oct 14, 2026
02db9b0
annotate_fastmath_flags: use more conservative afn flag
tkarna Oct 14, 2026
d4b2ae9
attention_fusion: simplify final clean up
tkarna Oct 14, 2026
b37d59c
transform_ext: add fold_scalar_or_splat_constant op
tkarna Oct 14, 2026
dac6157
fused_attention_schedule: ise fold_scalar_or_splat_constant op
tkarna Oct 14, 2026
db24299
transform_ext: add narrow_vector_contract_operands op
tkarna Oct 9, 2026
a0cfe18
lowering_common: vectorize applies vector.contract operand narrowing
tkarna Oct 9, 2026
56fb538
kernel_bench: force torch-mlir SDPB math backend to enable lowering t…
tkarna Oct 9, 2026
ca282b7
kernel_bench: tweak attention q load tile to enable lowering
tkarna Oct 14, 2026
f8f5307
pyproject.toml: update llvm version
tkarna Oct 14, 2026
d9dd187
fused_attention_schedule: tweak comment
tkarna Oct 14, 2026
68e7f4a
attention_fusion: fix producer fusion for non-masked case
tkarna Oct 14, 2026
180bace
fused_attention_schedule cleanup
tkarna Oct 14, 2026
950361f
lowering_common: vectorize includes rm-dead-code pass
tkarna Oct 14, 2026
dc8e2aa
fused_attention_schedule: fold constants and add fastmath in reductio…
tkarna Oct 14, 2026
6c0986d
cleanup_schedule: replace linalg elemwise fuse pass with cleanup patt…
tkarna Oct 14, 2026
b394c79
wg_tiling_schedule: update to work with new flash attn transform
tkarna Oct 14, 2026
cd7c356
vectorize_schedule: remove-dead-values pass is in vector helper now
tkarna Oct 14, 2026
f8de8fb
annotate_layouts_schedule: tweak attention q load tile size
tkarna Oct 14, 2026
55606d7
mlp_schedule: fix gemm k tiling
tkarna Oct 9, 2026
7cabed0
wg_tiling_schedule: fixes for gemm/reduction/attention cases
tkarna Oct 9, 2026
445f816
fix torch llama example
tkarna Oct 12, 2026
70e9678
mlp_schedule: fix MLP dpas annotation
tkarna Oct 15, 2026
6d06898
annotate_layouts_schedule: fix MLP dpas layout annotation
tkarna Oct 15, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 7 additions & 3 deletions examples/llama/ref_model.py
Original file line number Diff line number Diff line change
Expand Up @@ -273,9 +273,13 @@ def forward(self, tokens: torch.Tensor, start_pos: int):
# only for the new sequence. Thus, the matrix of scores is of size
# (seqlen, cache_len + seqlen), and the only masked entries are (i, j) for
# j > cache_len + i, since row i corresponds to token cache_len + i.
mask = torch.hstack(
[torch.zeros((seqlen, start_pos), device=tokens.device), mask]
).type_as(h)
# Skip the empty (seqlen, 0) prefix when start_pos == 0: torch-mlir
# cannot lower the resulting zero-sized aten.empty_strided.
if start_pos > 0:
mask = torch.hstack(
[torch.zeros((seqlen, start_pos), device=tokens.device), mask]
)
mask = mask.type_as(h)

for layer in self.layers:
h = layer(h, start_pos, freqs_cis, mask)
Expand Down
9 changes: 7 additions & 2 deletions examples/xegpu/kernel_bench.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@

import torch
import torch._dynamo as dynamo
from torch.nn.attention import sdpa_kernel, SDPBackend
import os
import gc
import numpy as np
Expand Down Expand Up @@ -284,7 +285,7 @@ def infer_parameters(
"sg_rows": 16,
"subgroup_size": 16,
"reduction_tile": 64,
"q_load_tile": [16, 32],
"q_load_tile": [16, 16],
"v_load_tile": [32, 32],
"prefetch_tile": [16, 32],
"nb_prefetch": 1,
Expand Down Expand Up @@ -727,7 +728,11 @@ def lower_and_execute_benchmark(
# BackendCompilerFailed) and we stop here; in execute mode it runs the
# compiled kernel on the device.
try:
with torch.no_grad():
# Force the math SDPA backend so attention is decomposed into
# supportable ops; on XPU the dispatcher otherwise selects the fused
# '_scaled_dot_product_fused_attention_overrideable' op that fails to
# legalize.
with torch.no_grad(), sdpa_kernel(SDPBackend.MATH):
result = torch_model(*torch_inputs)
except TypeError as e:
msg = str(e)
Expand Down
8 changes: 8 additions & 0 deletions lighthouse/dialects/transform/transform_ext/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
from .ops.param_cmp_eq import param_cmp_eq
from .ops.replace import replace
from .ops.convert_func_results_to_args import convert_func_results_to_args
from .ops.narrow_vector_contract_operands import narrow_vector_contract_operands
from .ops.emit_definite_failure import emit_definite_failure
from .ops.extract_handle import extract_handle
from .ops.get_tileable_consumers import get_tileable_consumers
Expand All @@ -22,10 +23,13 @@
from .ops.filter_num_loops import filter_num_loops
from .ops.filter_elementwise import filter_elementwise
from .ops.fuse_elementwise_op import fuse_elementwise_op
from .ops.fold_reshape_ops_by_expansion import fold_reshape_ops_by_expansion
from .ops.fold_scalar_or_splat_constant import fold_scalar_or_splat_constant
from .ops.filter_by_name import filter_by_name
from .ops.filter_reduction_ops import filter_reduction_ops
from .ops.fuse_dependent_reduction_ops import fuse_dependent_reduction_ops
from .ops.filter_non_contraction_reductions import filter_non_contraction_reductions
from .ops.rewrite_attention_row_mask import rewrite_attention_row_mask
from .ops.filter_contraction_ops import filter_contraction_ops
from .ops.get_leading_unit_tile_sizes import get_leading_unit_tile_sizes
from .ops.move_offsets_to_subview import move_offsets_to_subview
Expand Down Expand Up @@ -57,6 +61,8 @@
"filter_non_contraction_reductions",
"filter_num_loops",
"filter_reduction_ops",
"fold_reshape_ops_by_expansion",
"fold_scalar_or_splat_constant",
"fuse_dependent_reduction_ops",
"fuse_elementwise_op",
"get_fusion_roots",
Expand All @@ -70,11 +76,13 @@
"infer_xegpu_gemm_params",
"infer_xegpu_reduction_params",
"move_offsets_to_subview",
"narrow_vector_contract_operands",
"param_cmp_eq",
"propagate_tile_sizes",
"register_and_load",
"replace",
"reverse_handles",
"rewrite_attention_row_mask",
"sfc_remap_forall",
"sink_extract_slice_into_loop",
"sink_normalization_past_contraction",
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,243 @@
"""Temporary static-tensor subset of MLIR's reshape-by-expansion patterns."""

from math import prod

from mlir import ir
from mlir.dialects import arith, ext, linalg, tensor, transform
from mlir.dialects.transform import DiagnosedSilenceableFailure

from lighthouse.dialects.transform.transform_ext import TransformExtensionDialect
from lighthouse.utils.mlir import indexing_maps, opview


def _producer(value):
return opview(value.owner) if isinstance(value, ir.OpResult) else None


def _groups(reshape):
return [[int(dim) for dim in group] for group in reshape.reassociation]


def _plan(generic, operand_number, reshape):
maps = indexing_maps(generic)
if (
not generic.outputs
or not all(isinstance(v.type, ir.RankedTensorType) for v in generic.outputs)
or any(not m.is_projected_permutation or m.n_symbols for m in maps)
or not maps[operand_number].results
or any(op.regions for op in generic.regions[0].blocks[0].operations)
):
return None

extents = [None] * len(generic.iterator_types)
for value, m in zip(generic.operands, maps):
ty = value.type
if not isinstance(ty, ir.RankedTensorType):
if isinstance(ty, ir.ShapedType) or m.results:
return None
continue
if ty.encoding is not None or any(size <= 0 for size in ty.shape):
return None
for expr, size in zip(m.results, ty.shape):
dim = expr.position
if extents[dim] is not None and extents[dim] != size:
return None
extents[dim] = size
if any(size is None for size in extents):
return None

expanded_type = (
reshape.src.type
if isinstance(reshape, tensor.CollapseShapeOp)
else reshape.result.type
)
if expanded_type.encoding is not None or any(s <= 0 for s in expanded_type.shape):
return None
shapes = [[size] for size in extents]
for expr, group in zip(maps[operand_number].results, _groups(reshape)):
shape = [expanded_type.shape[i] for i in group]
if prod(shape) != extents[expr.position]:
return None
shapes[expr.position] = shape
if all(len(shape) == 1 for shape in shapes):
return None

loop_groups = []
num_loops = 0
for shape in shapes:
loop_groups.append(list(range(num_loops, num_loops + len(shape))))
num_loops += len(shape)
return maps, shapes, loop_groups, num_loops


def _expand(value, shape, groups):
ty = value.type
if not isinstance(ty, ir.RankedTensorType) or list(ty.shape) == shape:
return value
expanded_type = ir.RankedTensorType.get(shape, ty.element_type)
producer = _producer(value)
if (
isinstance(producer, tensor.CollapseShapeOp)
and producer.src.type == expanded_type
and _groups(producer) == groups
):
return producer.src
return tensor.ExpandShapeOp(expanded_type, value, groups, [], shape).result


def _rewrite(generic, reshape, plan, rewriter):
maps, shapes, loop_groups, num_loops = plan
with generic.location, ir.InsertionPoint(generic):
operands, new_maps, reassociations = [], [], []
for value, m in zip(generic.operands, maps):
shape, groups, exprs = [], [], []
for expr in m.results:
sizes = shapes[expr.position]
groups.append(list(range(len(shape), len(shape) + len(sizes))))
shape.extend(sizes)
exprs.extend(
ir.AffineExpr.get_dim(d) for d in loop_groups[expr.position]
)
operands.append(_expand(value, shape, groups))
new_maps.append(ir.AffineMapAttr.get(ir.AffineMap.get(num_loops, 0, exprs)))
reassociations.append(groups)

expanded = opview(generic.operation.clone())
for i, value in enumerate(operands):
expanded.operands[i] = value
expanded.attributes["indexing_maps"] = ir.ArrayAttr.get(new_maps)
expanded.attributes["iterator_types"] = ir.ArrayAttr.get(
[it for it, shape in zip(generic.iterator_types, shapes) for _ in shape]
)
for result, output in zip(expanded.results, expanded.outputs):
result.set_type(output.type)

# An original index denotes the linearized coordinate, not just the
# first of the expanded loop dimensions.
for index in list(expanded.regions[0].blocks[0].operations):
if not isinstance(index, linalg.IndexOp):
continue
dim = int(index.dim)
with ir.InsertionPoint(index):
coord = linalg.IndexOp(loop_groups[dim][0]).result
for new_dim, size in zip(loop_groups[dim][1:], shapes[dim][1:]):
stride = arith.ConstantOp(ir.IndexType.get(), size)
coord = arith.AddIOp(
arith.MulIOp(coord, stride.result).result,
linalg.IndexOp(new_dim).result,
).result
rewriter.replace_op(index, [coord])

replacements = []
for i, (old, new) in enumerate(zip(generic.results, expanded.results)):
if old.type == new.type:
replacements.append(new)
else:
replacements.append(
tensor.CollapseShapeOp(
old.type, new, reassociations[len(generic.inputs) + i]
).result
)

if isinstance(reshape, tensor.ExpandShapeOp):
result_number = reshape.src.result_number
rewriter.replace_op(reshape, [expanded.results[result_number]])
rewriter.replace_op(generic, replacements)


def _fold_one(root, rewriter):
candidates = []

def visit(operation):
if operation.name in ("linalg.generic", "tensor.expand_shape"):
candidates.append(operation.opview)
return ir.WalkResult.ADVANCE

root.walk(visit)
for candidate in candidates:
if isinstance(candidate, linalg.GenericOp):
for i, value in enumerate(candidate.inputs):
reshape = _producer(value)
if not isinstance(reshape, tensor.CollapseShapeOp):
continue
plan = _plan(candidate, i, reshape)
if plan is not None:
_rewrite(candidate, reshape, plan, rewriter)
return True
else:
generic = _producer(candidate.src)
# Do not rewrite a producer outside the requested container.
if not isinstance(generic, linalg.GenericOp) or generic not in candidates:
continue
i = len(generic.inputs) + candidate.src.result_number
plan = _plan(generic, i, candidate)
if plan is not None:
_rewrite(generic, candidate, plan, rewriter)
return True
return False


class FoldReshapeOpsByExpansionOp(
TransformExtensionDialect.Operation, name="fold_reshape_ops_by_expansion"
):
"""Fold tensor reshapes into generic loop dimensions without body fusion.
Temporary replacement for the not-yet-exposed native pattern collection.
Within one func.func or builtin.module, greedily fold collapse_shape inputs
and expand_shape results of linalg.generic. All legal matches are accepted,
including shared producers; collapsed results preserve other users.
Supports positive static, unencoded tensors, projected-permutation indexing
maps, scalar inputs and region-free bodies (including linalg.index).
Dynamic/encoded tensors, non-projected maps, named ops, nested scalar regions
and the native pad patterns are not implemented and are left unchanged.
Run canonicalization afterwards to clean up introduced reshapes.
Consumes the container handle and invalidates nested handles. Reacquire
payload handles through the returned container after this transform.
"""

target: ext.Operand[transform.AnyOpType]
result: ext.Result[transform.AnyOpType[()]] = ext.infer_result()

@classmethod
def attach_interface_impls(cls, context=None):
cls.TransformOpInterfaceModel.attach(cls.OPERATION_NAME, context=context)
cls.MemoryEffectsOpInterfaceModel.attach(cls.OPERATION_NAME, context=context)

class TransformOpInterfaceModel(transform.TransformOpInterface):
@staticmethod
def apply(op, rewriter, results, state) -> DiagnosedSilenceableFailure:
targets = [target.operation for target in state.get_payload_ops(op.target)]
if len(targets) != 1 or targets[0].name not in (
"func.func",
"builtin.module",
):
op.operation.location.emit_error(
"fold_reshape_ops_by_expansion expects one func.func or builtin.module"
)
return DiagnosedSilenceableFailure.SilenceableFailure
while _fold_one(targets[0], rewriter):
pass
results.set_ops(op.result, targets)
return DiagnosedSilenceableFailure.Success

@staticmethod
def allow_repeated_handle_operands(_op) -> bool:
return False

class MemoryEffectsOpInterfaceModel(ir.MemoryEffectsOpInterface):
@staticmethod
def get_effects(op):
return (
transform.consumes_handle(op.op_operands)
+ transform.produces_handle(op.results)
+ transform.modifies_payload()
)


def fold_reshape_ops_by_expansion(
target: ir.Value[transform.AnyOpType],
) -> ir.Value[transform.AnyOpType]:
"""Expand generic iteration spaces across static tensor reshape boundaries."""
return FoldReshapeOpsByExpansionOp(target=target).result
Loading
Loading