Skip to content

Commit b1c3093

Browse files
committed
Merge branch 'main' into attention_perf_regression_fix
2 parents 00e8e33 + e207770 commit b1c3093

60 files changed

Lines changed: 5686 additions & 416 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎examples/llama/ref_model.py‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -273,9 +273,13 @@ def forward(self, tokens: torch.Tensor, start_pos: int):
273273
# only for the new sequence. Thus, the matrix of scores is of size
274274
# (seqlen, cache_len + seqlen), and the only masked entries are (i, j) for
275275
# j > cache_len + i, since row i corresponds to token cache_len + i.
276-
mask = torch.hstack(
277-
[torch.zeros((seqlen, start_pos), device=tokens.device), mask]
278-
).type_as(h)
276+
# Skip the empty (seqlen, 0) prefix when start_pos == 0: torch-mlir
277+
# cannot lower the resulting zero-sized aten.empty_strided.
278+
if start_pos > 0:
279+
mask = torch.hstack(
280+
[torch.zeros((seqlen, start_pos), device=tokens.device), mask]
281+
)
282+
mask = mask.type_as(h)
279283

280284
for layer in self.layers:
281285
h = layer(h, start_pos, freqs_cis, mask)

‎examples/xegpu/kernel_bench.py‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@
5151

5252
import torch
5353
import torch._dynamo as dynamo
54+
from torch.nn.attention import sdpa_kernel, SDPBackend
5455
import os
5556
import gc
5657
import numpy as np
@@ -284,7 +285,7 @@ def infer_parameters(
284285
"sg_rows": 16,
285286
"subgroup_size": 16,
286287
"reduction_tile": 64,
287-
"q_load_tile": [16, 32],
288+
"q_load_tile": [16, 16],
288289
"v_load_tile": [32, 32],
289290
"prefetch_tile": [16, 32],
290291
"nb_prefetch": 1,
@@ -727,7 +728,11 @@ def lower_and_execute_benchmark(
727728
# BackendCompilerFailed) and we stop here; in execute mode it runs the
728729
# compiled kernel on the device.
729730
try:
730-
with torch.no_grad():
731+
# Force the math SDPA backend so attention is decomposed into
732+
# supportable ops; on XPU the dispatcher otherwise selects the fused
733+
# '_scaled_dot_product_fused_attention_overrideable' op that fails to
734+
# legalize.
735+
with torch.no_grad(), sdpa_kernel(SDPBackend.MATH):
731736
result = torch_model(*torch_inputs)
732737
except TypeError as e:
733738
msg = str(e)

‎lighthouse/dialects/transform/transform_ext/__init__.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
from .ops.param_cmp_eq import param_cmp_eq
1010
from .ops.replace import replace
1111
from .ops.convert_func_results_to_args import convert_func_results_to_args
12+
from .ops.narrow_vector_contract_operands import narrow_vector_contract_operands
1213
from .ops.emit_definite_failure import emit_definite_failure
1314
from .ops.extract_handle import extract_handle
1415
from .ops.get_tileable_consumers import get_tileable_consumers
@@ -22,9 +23,13 @@
2223
from .ops.filter_num_loops import filter_num_loops
2324
from .ops.filter_elementwise import filter_elementwise
2425
from .ops.fuse_elementwise_op import fuse_elementwise_op
26+
from .ops.fold_reshape_ops_by_expansion import fold_reshape_ops_by_expansion
27+
from .ops.fold_scalar_or_splat_constant import fold_scalar_or_splat_constant
2528
from .ops.filter_by_name import filter_by_name
2629
from .ops.filter_reduction_ops import filter_reduction_ops
2730
from .ops.fuse_dependent_reduction_ops import fuse_dependent_reduction_ops
31+
from .ops.filter_non_contraction_reductions import filter_non_contraction_reductions
32+
from .ops.rewrite_attention_row_mask import rewrite_attention_row_mask
2833
from .ops.filter_contraction_ops import filter_contraction_ops
2934
from .ops.get_leading_unit_tile_sizes import get_leading_unit_tile_sizes
3035
from .ops.move_offsets_to_subview import move_offsets_to_subview
@@ -53,8 +58,11 @@
5358
"filter_by_name",
5459
"filter_contraction_ops",
5560
"filter_elementwise",
61+
"filter_non_contraction_reductions",
5662
"filter_num_loops",
5763
"filter_reduction_ops",
64+
"fold_reshape_ops_by_expansion",
65+
"fold_scalar_or_splat_constant",
5866
"fuse_dependent_reduction_ops",
5967
"fuse_elementwise_op",
6068
"get_fusion_roots",
@@ -68,11 +76,13 @@
6876
"infer_xegpu_gemm_params",
6977
"infer_xegpu_reduction_params",
7078
"move_offsets_to_subview",
79+
"narrow_vector_contract_operands",
7180
"param_cmp_eq",
7281
"propagate_tile_sizes",
7382
"register_and_load",
7483
"replace",
7584
"reverse_handles",
85+
"rewrite_attention_row_mask",
7686
"sfc_remap_forall",
7787
"sink_extract_slice_into_loop",
7888
"sink_normalization_past_contraction",

‎lighthouse/dialects/transform/transform_ext/ops/filter_contraction_ops.py‎

Lines changed: 2 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -4,45 +4,7 @@
44
from lighthouse.dialects.transform.transform_ext.utils.make_filter_handles_op import (
55
make_filter_handles_op,
66
)
7-
from lighthouse.utils.mlir import opview, indexing_maps, linalg_inputs, dim_position
8-
9-
10-
def _map_dims(m: ir.AffineMap) -> set[int]:
11-
"""Set of iteration-dim positions referenced by an affine map's results."""
12-
dims = set()
13-
for r in m.results:
14-
pos = dim_position(r)
15-
if pos is not None:
16-
dims.add(pos)
17-
return dims
18-
19-
20-
def _is_structural_contraction(ov: ir.OpView) -> bool:
21-
"""Detect a matmul-like op from its indexing maps alone (body-agnostic).
22-
23-
A contraction has a reduction dimension (one that is absent from the
24-
output) that is shared by at least two inputs. This ignores the body, so it
25-
also matches contractions with extra elementwise operands (e.g. a scaled or
26-
dequantized input) that the strict multiply-accumulate matcher rejects.
27-
"""
28-
maps = indexing_maps(ov)
29-
inputs = linalg_inputs(ov)
30-
if not maps or not inputs:
31-
return False
32-
num_inputs = len(inputs)
33-
input_maps = maps[:num_inputs]
34-
output_maps = maps[num_inputs:]
35-
36-
out_dims: set[int] = set()
37-
for m in output_maps:
38-
out_dims |= _map_dims(m)
39-
40-
# Reduction dims are the iteration dims not present in any output.
41-
reduction_dims = set(range(maps[0].n_dims)) - out_dims
42-
for k in reduction_dims:
43-
if sum(1 for m in input_maps if k in _map_dims(m)) >= 2:
44-
return True
45-
return False
7+
from lighthouse.utils.mlir import opview, is_structural_contraction
468

479

4810
def is_contraction_op(op: ir.Operation | ir.OpView) -> bool:
@@ -56,7 +18,7 @@ def is_contraction_op(op: ir.Operation | ir.OpView) -> bool:
5618
ov = opview(op)
5719
if "linalg" not in ov.operation.name:
5820
return False
59-
return linalg.isa_contraction_op(ov) or _is_structural_contraction(ov)
21+
return linalg.isa_contraction_op(ov) or is_structural_contraction(ov)
6022

6123

6224
FilterContractionOpsOp = make_filter_handles_op(
Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
from mlir import ir
2+
from mlir.dialects import transform
3+
4+
from lighthouse.dialects.transform.transform_ext.utils.make_filter_handles_op import (
5+
make_filter_handles_op,
6+
)
7+
from lighthouse.utils.mlir import is_linalg_reduction_op
8+
9+
FilterNonContractionReductionsOp = make_filter_handles_op(
10+
"filter_non_contraction_reductions", is_linalg_reduction_op
11+
)
12+
13+
14+
def filter_non_contraction_reductions(
15+
target: ir.Value[transform.AnyOpType],
16+
) -> ir.Value:
17+
"""
18+
snake_case wrapper to create a FilterNonContractionReductionsOp.
19+
20+
Keeps single-output linalg reductions (e.g. row max/sum, norms) that are
21+
not contractions, convolutions or pooling ops.
22+
23+
Args:
24+
target: Handle to target op(s).
25+
Returns:
26+
Handle to the non-contraction reduction subset of `target`.
27+
"""
28+
return FilterNonContractionReductionsOp(target=target).ops

0 commit comments

Comments
 (0)