Skip to content

Commit da54528

Browse files
authored
[x86] Register tiling for non-contraction reductions (#296)
Adds a target-aware heuristic used by the register parallel, reduction and unroll strategies to drive tiling on non-contraction reductions. General reductions were never tiling anchors, so they only received propagated parallel tiles and vectorized into full-sized multi-reductions. Row and column reductions now vectorize into register-sized chunks with independent accumulators, avoiding register spills. Assisted-by: Claude
1 parent fd96b72 commit da54528

29 files changed

Lines changed: 1384 additions & 122 deletions

‎lighthouse/dialects/transform/transform_ext/__init__.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
from .ops.filter_by_name import filter_by_name
2626
from .ops.filter_reduction_ops import filter_reduction_ops
2727
from .ops.fuse_dependent_reduction_ops import fuse_dependent_reduction_ops
28+
from .ops.filter_non_contraction_reductions import filter_non_contraction_reductions
2829
from .ops.filter_contraction_ops import filter_contraction_ops
2930
from .ops.get_leading_unit_tile_sizes import get_leading_unit_tile_sizes
3031
from .ops.move_offsets_to_subview import move_offsets_to_subview
@@ -53,6 +54,7 @@
5354
"filter_by_name",
5455
"filter_contraction_ops",
5556
"filter_elementwise",
57+
"filter_non_contraction_reductions",
5658
"filter_num_loops",
5759
"filter_reduction_ops",
5860
"fuse_dependent_reduction_ops",

‎lighthouse/dialects/transform/transform_ext/ops/filter_contraction_ops.py‎

Lines changed: 2 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -4,45 +4,7 @@
44
from lighthouse.dialects.transform.transform_ext.utils.make_filter_handles_op import (
55
make_filter_handles_op,
66
)
7-
from lighthouse.utils.mlir import opview, indexing_maps, linalg_inputs, dim_position
8-
9-
10-
def _map_dims(m: ir.AffineMap) -> set[int]:
11-
"""Set of iteration-dim positions referenced by an affine map's results."""
12-
dims = set()
13-
for r in m.results:
14-
pos = dim_position(r)
15-
if pos is not None:
16-
dims.add(pos)
17-
return dims
18-
19-
20-
def _is_structural_contraction(ov: ir.OpView) -> bool:
21-
"""Detect a matmul-like op from its indexing maps alone (body-agnostic).
22-
23-
A contraction has a reduction dimension (one that is absent from the
24-
output) that is shared by at least two inputs. This ignores the body, so it
25-
also matches contractions with extra elementwise operands (e.g. a scaled or
26-
dequantized input) that the strict multiply-accumulate matcher rejects.
27-
"""
28-
maps = indexing_maps(ov)
29-
inputs = linalg_inputs(ov)
30-
if not maps or not inputs:
31-
return False
32-
num_inputs = len(inputs)
33-
input_maps = maps[:num_inputs]
34-
output_maps = maps[num_inputs:]
35-
36-
out_dims: set[int] = set()
37-
for m in output_maps:
38-
out_dims |= _map_dims(m)
39-
40-
# Reduction dims are the iteration dims not present in any output.
41-
reduction_dims = set(range(maps[0].n_dims)) - out_dims
42-
for k in reduction_dims:
43-
if sum(1 for m in input_maps if k in _map_dims(m)) >= 2:
44-
return True
45-
return False
7+
from lighthouse.utils.mlir import opview, is_structural_contraction
468

479

4810
def is_contraction_op(op: ir.Operation | ir.OpView) -> bool:
@@ -56,7 +18,7 @@ def is_contraction_op(op: ir.Operation | ir.OpView) -> bool:
5618
ov = opview(op)
5719
if "linalg" not in ov.operation.name:
5820
return False
59-
return linalg.isa_contraction_op(ov) or _is_structural_contraction(ov)
21+
return linalg.isa_contraction_op(ov) or is_structural_contraction(ov)
6022

6123

6224
FilterContractionOpsOp = make_filter_handles_op(
Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
from mlir import ir
2+
from mlir.dialects import transform
3+
4+
from lighthouse.dialects.transform.transform_ext.utils.make_filter_handles_op import (
5+
make_filter_handles_op,
6+
)
7+
from lighthouse.utils.mlir import is_linalg_reduction_op
8+
9+
FilterNonContractionReductionsOp = make_filter_handles_op(
10+
"filter_non_contraction_reductions", is_linalg_reduction_op
11+
)
12+
13+
14+
def filter_non_contraction_reductions(
15+
target: ir.Value[transform.AnyOpType],
16+
) -> ir.Value:
17+
"""
18+
snake_case wrapper to create a FilterNonContractionReductionsOp.
19+
20+
Keeps single-output linalg reductions (e.g. row max/sum, norms) that are
21+
not contractions, convolutions or pooling ops.
22+
23+
Args:
24+
target: Handle to target op(s).
25+
Returns:
26+
Handle to the non-contraction reduction subset of `target`.
27+
"""
28+
return FilterNonContractionReductionsOp(target=target).ops

‎lighthouse/dialects/transform/transform_ext/ops/get_fusion_roots.py‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
from lighthouse.dialects.transform.transform_ext.utils import tile_size_analysis as tsa
77
from lighthouse.dialects.transform.transform_ext.utils import fusion_analysis as fa
88
from lighthouse.dialects.transform.transform_ext.utils import tile_propagation as tp
9-
from lighthouse.utils.mlir import op_users
9+
from lighthouse.utils.mlir import is_linalg_reduction_op, op_users
1010

1111

1212
class GetFusionRootsOp(TransformExtensionDialect.Operation, name="get_fusion_roots"):
@@ -48,11 +48,18 @@ def _same_group_consumer(
4848
"""Check whether `consumer` shares `producer`'s fusion group across `shared` tensor."""
4949
if fa.is_fusion_boundary(consumer):
5050
return False
51+
consumer_sizes = tsa.get_tile_sizes_attr(consumer)
52+
# A reduction fused into a consumer tiled along dims its result is
53+
# broadcast over would be recomputed per tile (a full pass each).
54+
if is_linalg_reduction_op(producer) and tp.tiles_beyond_value(
55+
consumer, consumer_sizes, shared
56+
):
57+
return False
5158
return tp.compatible_on_value(
5259
producer,
5360
tsa.get_tile_sizes_attr(producer),
5461
consumer,
55-
tsa.get_tile_sizes_attr(consumer),
62+
consumer_sizes,
5663
shared,
5764
)
5865

‎lighthouse/dialects/transform/transform_ext/utils/tile_propagation.py‎

Lines changed: 30 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
opview,
77
indexing_maps,
88
dim_position,
9+
is_linalg_reduction_op,
910
linalg_inputs,
1011
linalg_outputs,
1112
)
@@ -15,15 +16,22 @@
1516
def is_propagatable(op: ir.Operation | ir.OpView) -> bool:
1617
"""Check whether tile sizes may be propagated onto this op.
1718
18-
True for any structured linalg op that is not a fusion barrier; non-linalg
19-
ops have no indexing maps to translate tiles through and are excluded.
19+
True for structured linalg ops that are neither fusion barriers nor
20+
non-contraction reductions. Non-linalg ops have no indexing maps to
21+
translate tiles through. Reductions are anchored by their own strategy, as
22+
tiles derived from neighbours (e.g. a GEMM register tile) can be a poor fit
23+
for them; their own tiles still propagate to their neighbours.
2024
2125
Args:
2226
op: Candidate op to annotate.
2327
Returns:
2428
True if `op` can receive propagated tile sizes.
2529
"""
26-
return indexing_maps(op) is not None and not fa.is_fusion_barrier(op)
30+
return (
31+
indexing_maps(op) is not None
32+
and not fa.is_fusion_barrier(op)
33+
and not is_linalg_reduction_op(op)
34+
)
2735

2836

2937
def _map_for_value(
@@ -93,6 +101,25 @@ def tiles_on_value(
93101
return tiles
94102

95103

104+
def tiles_beyond_value(
105+
op: ir.Operation | ir.OpView, sizes: Sequence[int], value: ir.Value
106+
) -> bool:
107+
"""Whether `op`, tiled by `sizes`, tiles a loop dim that `value` does not
108+
index (e.g. the columns of a consumer of a row reduction's result).
109+
110+
Fused into such a tile loop, the producer of `value` is recomputed per tile.
111+
"""
112+
ov = opview(op)
113+
maps = indexing_maps(ov)
114+
if maps is None:
115+
return False
116+
value_map = _map_for_value(ov, value, maps)
117+
if value_map is None:
118+
return False
119+
indexed = {dim_position(expr) for expr in value_map.results}
120+
return any(size and dim not in indexed for dim, size in enumerate(sizes))
121+
122+
96123
def compatible_on_value(
97124
src_op: ir.Operation | ir.OpView,
98125
src_sizes: Sequence[int],

‎lighthouse/dialects/transform/transform_ext/utils/tiling/common.py‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,25 @@ def largest_divisor(dim_size: int | None, limit: int) -> int:
2525
return 1
2626

2727

28+
def largest_multiple_divisor(
29+
dim_size: int | None, limit: int, multiple_of: int
30+
) -> int | None:
31+
"""Largest multiple of `multiple_of` <= `limit` dividing `dim_size`.
32+
33+
Sizes not above `limit` are taken whole; None when nothing divides.
34+
"""
35+
if dim_size is None:
36+
return None
37+
if dim_size <= limit:
38+
return dim_size
39+
tile = (limit // multiple_of) * multiple_of
40+
while tile >= multiple_of:
41+
if dim_size % tile == 0:
42+
return tile
43+
tile -= multiple_of
44+
return None
45+
46+
2847
def assign_from_tail(dims: list[int], values: list[int], sizes: list[int]) -> None:
2948
"""Write `values` onto `sizes` at the trailing `dims`, aligning both by their tails."""
3049
if not dims or not values:
Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
from typing import NamedTuple
2+
3+
from mlir import ir
4+
5+
from lighthouse.utils.mlir import (
6+
dim_position,
7+
indexing_maps,
8+
is_linalg_reduction_op,
9+
linalg_inputs,
10+
linalg_loop_dim_sizes,
11+
linalg_outputs,
12+
linalg_reduction_dims,
13+
opview,
14+
)
15+
16+
17+
class ReductionInfo(NamedTuple):
18+
"""Structure of a non-contraction reduction relevant to vectorization.
19+
20+
Reductions are classified by their vector dim, i.e. the loop dim indexing
21+
the innermost (contiguous) dim of the highest-rank input:
22+
* inner: vector dim is reduced (e.g. row max/sum), lanes are
23+
combined by a horizontal reduction,
24+
* outer: vector dim is parallel (e.g. RMSNorm channel sum), every lane is
25+
an independent accumulator and no horizontal reduction is needed.
26+
"""
27+
28+
vector_dim: int
29+
dim_sizes: list[int | None]
30+
parallel_dims: list[int]
31+
reduction_dims: list[int]
32+
inner: bool
33+
elem_type: ir.Type
34+
35+
36+
def _reduction_vector_dim(op: ir.OpView) -> int | None:
37+
"""Loop dim indexing the innermost dim of the highest-rank input."""
38+
best_map = None
39+
for value, amap in zip(linalg_inputs(op), indexing_maps(op)):
40+
if not isinstance(value.type, ir.ShapedType) or not amap.results:
41+
continue
42+
if best_map is None or len(amap.results) > len(best_map.results):
43+
best_map = amap
44+
if best_map is None:
45+
return None
46+
return dim_position(best_map.results[-1])
47+
48+
49+
def reduction_info(op: ir.Operation | ir.OpView) -> ReductionInfo | None:
50+
"""Analyze a non-contraction reduction, else None."""
51+
ov = opview(op)
52+
if not is_linalg_reduction_op(ov):
53+
return None
54+
vector_dim = _reduction_vector_dim(ov)
55+
if vector_dim is None:
56+
return None
57+
dim_sizes = linalg_loop_dim_sizes(ov)
58+
reduction_dims = linalg_reduction_dims(ov)
59+
return ReductionInfo(
60+
vector_dim=vector_dim,
61+
dim_sizes=dim_sizes,
62+
parallel_dims=[d for d in range(len(dim_sizes)) if d not in reduction_dims],
63+
reduction_dims=reduction_dims,
64+
inner=vector_dim in reduction_dims,
65+
elem_type=ir.ShapedType(linalg_outputs(ov)[0].type).element_type,
66+
)

‎lighthouse/dialects/transform/transform_ext/utils/tiling/strategy_cache.py‎

Lines changed: 27 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,16 @@
22
from mlir.dialects import linalg
33

44
from lighthouse.execution.target import TargetInfo
5-
from lighthouse.utils.mlir import linalg_outputs, opview, is_linalg_eltwise_op
5+
from lighthouse.utils.mlir import (
6+
is_linalg_eltwise_op,
7+
is_linalg_reduction_op,
8+
linalg_outputs,
9+
opview,
10+
)
611

712
from .strategy_base import StrategyContext, TilingStrategy
813
from .common import disable_small_tiles, parallel_and_reduction_dims
9-
from .target_caps import vector_lane_count
14+
from .target_caps import storage_lane_count
1015

1116

1217
class EltwiseCacheTiling:
@@ -150,7 +155,7 @@ def choose_parallel_tile_shape(
150155
fast_axis = rank - 1
151156

152157
tiles = [1] * rank
153-
vector_width = vector_lane_count(target, out_type.element_type)
158+
vector_width = storage_lane_count(target, out_type.element_type)
154159
fast_extent = planning_shape[fast_axis]
155160
candidate_fast = max(
156161
vector_width,
@@ -196,6 +201,23 @@ class CacheTilingStrategy(TilingStrategy):
196201

197202
_PARALLEL_TILE_DIMS = 2
198203

204+
@staticmethod
205+
def _reduction_op_tiles(op: ir.OpView, ctx: StrategyContext) -> list[int] | None:
206+
"""Cache tiles of a non-contraction reduction; reduced dims stay whole.
207+
208+
Parallel dims get the tiles of an elementwise op over the result.
209+
"""
210+
out_map = TilingStrategy.output_map(op)
211+
if out_map is None:
212+
return None
213+
tiles = EltwiseCacheTiling.choose_parallel_tile_shape(op, ctx.target)
214+
sizes = [0] * out_map.n_dims
215+
for expr, tile in zip(out_map.results, tiles):
216+
if isinstance(expr, ir.AffineDimExpr):
217+
sizes[expr.position] = tile
218+
disable_small_tiles(op, out_map, sizes, ctx.tile_size)
219+
return sizes
220+
199221
def compute(
200222
self, op: ir.Operation | ir.OpView, ctx: StrategyContext
201223
) -> list[int] | None:
@@ -222,6 +244,8 @@ def compute(
222244
sizes[dim] = value
223245
disable_small_tiles(ov, out_map, sizes, ctx.tile_size)
224246
return sizes
247+
if is_linalg_reduction_op(ov):
248+
return self._reduction_op_tiles(ov, ctx)
225249

226250
out_map = self.output_map(ov)
227251
if out_map is None:

0 commit comments

Comments
 (0)