Skip to content

Commit acbe06d

Browse files
authored
[descriptors][x86] Lower vector multi-reductions (#287)
Adds a schedule that lowers vector multi-reductions, and enables reassociation of floating-point reductions to allow LLVM lowering to use SIMD operations. It is in-line with the general assumption that fast-math is the default mode of operation. Assisted-by: Claude
1 parent 28c570f commit acbe06d

5 files changed

Lines changed: 136 additions & 1 deletion

File tree

‎lighthouse/pipeline/descriptors/llvm-lowering.yaml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,11 @@ Pipeline:
66
- pass: "lower-affine"
77
- pass: "finalize-memref-to-llvm"
88
- pass: "convert-scf-to-cf"
9-
- pass: "convert-vector-to-llvm{vector-contract-lowering=outerproduct}"
9+
- pass: "convert-vector-to-llvm{
10+
vector-contract-lowering=outerproduct
11+
reassociate-fp-reductions=1
12+
}"
13+
- pass: "math-expand-ops{ops=fpowi}"
1014
- pass: "convert-math-to-llvm"
1115
- pass: "convert-math-to-libm"
1216
- pass: "convert-to-llvm"

‎lighthouse/pipeline/descriptors/x86_64/lower.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ Pipeline:
99
- schedule: "x86/amx_move_offsets.py[gen=amx_move_offsets]"
1010
- schedule: "vectorization.py[gen=x86_vectorization]"
1111
- schedule: "vectorization.py[gen=vectorize_all]"
12+
- schedule: "vectorization.py[gen=lower_vector_multi_reductions]"
1213
- schedule: "vectorization.py[gen=flatten_vector_ops]"
1314

1415
## Lower to LLVM

‎lighthouse/schedule/__init__.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
from .tiling import tile_ops
99
from .vectorization import flatten_vector_ops
1010
from .vectorization import simplify_vector_ops
11+
from .vectorization import lower_vector_multi_reductions
1112
from .vectorization import vectorize_linalg
1213
from .vectorization import vectorize_all
1314
from .vectorization import x86_vectorization
@@ -24,6 +25,7 @@
2425
"flatten_vector_ops",
2526
"hoist_loops",
2627
"linalg_contract_fold_unit_dims",
28+
"lower_vector_multi_reductions",
2729
"parallelize",
2830
"print_ir",
2931
"schedule_boilerplate",

‎lighthouse/schedule/vectorization.py‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,54 @@ def vectorize_all() -> ir.Module:
5050
return schedule
5151

5252

53+
def lower_vector_multi_reductions() -> ir.Module:
54+
"""
55+
Lower vector.multi_reduction ops where possible.
56+
57+
Reductions over the innermost dim become horizontal `vector.reduction`s and
58+
reductions over outer dims become lane-wise elementwise ops. Leftovers
59+
(e.g. a reduced middle dim) are first reordered to reduce outer dims.
60+
61+
Returns:
62+
Schedule
63+
"""
64+
strategies = [
65+
vector.VectorMultiReductionLowering.InnerReduction,
66+
vector.VectorMultiReductionLowering.InnerParallel,
67+
]
68+
with schedule_boilerplate() as (schedule, named_seq):
69+
# First lower reductions already matching a strategy.
70+
with ir.InsertionPoint(
71+
transform.ApplyPatternsOp(named_seq.bodyTarget).patterns
72+
):
73+
for strategy in strategies:
74+
vector.apply_patterns_vector_multi_reduction_flattening(
75+
lowering_strategy=strategy
76+
)
77+
vector.apply_patterns_vector_multi_reduction_unrolling(
78+
lowering_strategy=strategy
79+
)
80+
# Reordering transposes every mismatched reduction, so run it on
81+
# the leftovers only, with one strategy only as using both strategies
82+
# creates conflicts.
83+
fallback = vector.VectorMultiReductionLowering.InnerParallel
84+
with ir.InsertionPoint(
85+
transform.ApplyPatternsOp(named_seq.bodyTarget).patterns
86+
):
87+
vector.apply_patterns_vector_reorder_multi_reduction_dims(
88+
lowering_strategy=fallback
89+
)
90+
vector.apply_patterns_vector_multi_reduction_flattening(
91+
lowering_strategy=fallback
92+
)
93+
vector.apply_patterns_vector_multi_reduction_unrolling(
94+
lowering_strategy=fallback
95+
)
96+
lh_transform.cleanup(named_seq.bodyTarget)
97+
transform.yield_()
98+
return schedule
99+
100+
53101
def x86_vectorization() -> ir.Module:
54102
"""
55103
Apply x86-specific vector rewrites.
Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
# RUN: %PYTHON %s | FileCheck %s
2+
3+
from mlir import ir
4+
5+
import lighthouse.dialects as lh_dialects
6+
from lighthouse.schedule import lower_vector_multi_reductions
7+
8+
9+
def run(name: str, payload_str: str):
10+
print(f"Test: {name}", flush=True)
11+
with ir.Context(), ir.Location.unknown():
12+
lh_dialects.register_and_load()
13+
payload = ir.Module.parse(payload_str)
14+
sched = lower_vector_multi_reductions()
15+
sched.body.operations[0].apply(payload.operation)
16+
print(payload)
17+
18+
19+
# Inner (row) reduction: one horizontal reduction per row, no transpose.
20+
# CHECK-LABEL: Test: row
21+
# CHECK-NOT: vector.transpose
22+
# CHECK-COUNT-2: vector.reduction <add>, %{{.*}}, %{{.*}} : vector<64xf32> into f32
23+
# CHECK-NOT: vector.multi_reduction
24+
run(
25+
"row",
26+
"""
27+
func.func @row(%a: vector<2x64xf32>, %acc: vector<2xf32>) -> vector<2xf32> {
28+
%0 = vector.multi_reduction <add>, %a, %acc [1] : vector<2x64xf32> to vector<2xf32>
29+
return %0 : vector<2xf32>
30+
}
31+
""",
32+
)
33+
34+
35+
# Outer (column) reduction: lane-wise adds, no transpose.
36+
# CHECK-LABEL: Test: column
37+
# CHECK-NOT: vector.transpose
38+
# CHECK-COUNT-2: arith.addf %{{.*}}, %{{.*}} : vector<32xf32>
39+
# CHECK-NOT: vector.multi_reduction
40+
run(
41+
"column",
42+
"""
43+
func.func @column(%a: vector<2x32xf32>, %acc: vector<32xf32>) -> vector<32xf32> {
44+
%0 = vector.multi_reduction <add>, %a, %acc [0] : vector<2x32xf32> to vector<32xf32>
45+
return %0 : vector<32xf32>
46+
}
47+
""",
48+
)
49+
50+
51+
# 1-D reduction lowers to a single horizontal reduction.
52+
# CHECK-LABEL: Test: rank1
53+
# CHECK: vector.reduction <maximumf>, %{{.*}}, %{{.*}} : vector<128xf32> into f32
54+
# CHECK-NOT: vector.multi_reduction
55+
run(
56+
"rank1",
57+
"""
58+
func.func @rank1(%a: vector<128xf32>, %acc: f32) -> f32 {
59+
%0 = vector.multi_reduction <maximumf>, %a, %acc [0] : vector<128xf32> to f32
60+
return %0 : f32
61+
}
62+
""",
63+
)
64+
65+
66+
# A reduced middle dim is not handled by either strategy directly and falls
67+
# back to reordering it outward.
68+
# CHECK-LABEL: Test: middle
69+
# CHECK: vector.transpose
70+
# CHECK-NOT: vector.multi_reduction
71+
# CHECK: return
72+
run(
73+
"middle",
74+
"""
75+
func.func @middle(%a: vector<2x4x16xf32>, %acc: vector<2x16xf32>) -> vector<2x16xf32> {
76+
%0 = vector.multi_reduction <add>, %a, %acc [1] : vector<2x4x16xf32> to vector<2x16xf32>
77+
return %0 : vector<2x16xf32>
78+
}
79+
""",
80+
)

0 commit comments

Comments
 (0)