Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion lighthouse/pipeline/descriptors/llvm-lowering.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,11 @@ Pipeline:
- pass: "lower-affine"
- pass: "finalize-memref-to-llvm"
- pass: "convert-scf-to-cf"
- pass: "convert-vector-to-llvm{vector-contract-lowering=outerproduct}"
- pass: "convert-vector-to-llvm{
vector-contract-lowering=outerproduct
reassociate-fp-reductions=1
}"
- pass: "math-expand-ops{ops=fpowi}"
- pass: "convert-math-to-llvm"
- pass: "convert-math-to-libm"
- pass: "convert-to-llvm"
Expand Down
1 change: 1 addition & 0 deletions lighthouse/pipeline/descriptors/x86_64/lower.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ Pipeline:
- schedule: "x86/amx_move_offsets.py[gen=amx_move_offsets]"
- schedule: "vectorization.py[gen=x86_vectorization]"
- schedule: "vectorization.py[gen=vectorize_all]"
- schedule: "vectorization.py[gen=lower_vector_multi_reductions]"
- schedule: "vectorization.py[gen=flatten_vector_ops]"

## Lower to LLVM
Expand Down
2 changes: 2 additions & 0 deletions lighthouse/schedule/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
from .tiling import tile_ops
from .vectorization import flatten_vector_ops
from .vectorization import simplify_vector_ops
from .vectorization import lower_vector_multi_reductions
from .vectorization import vectorize_linalg
from .vectorization import vectorize_all
from .vectorization import x86_vectorization
Expand All @@ -24,6 +25,7 @@
"flatten_vector_ops",
"hoist_loops",
"linalg_contract_fold_unit_dims",
"lower_vector_multi_reductions",
"parallelize",
"print_ir",
"schedule_boilerplate",
Expand Down
48 changes: 48 additions & 0 deletions lighthouse/schedule/vectorization.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,54 @@ def vectorize_all() -> ir.Module:
return schedule


def lower_vector_multi_reductions() -> ir.Module:
"""
Lower vector.multi_reduction ops where possible.
Reductions over the innermost dim become horizontal `vector.reduction`s and
reductions over outer dims become lane-wise elementwise ops. Leftovers
(e.g. a reduced middle dim) are first reordered to reduce outer dims.
Returns:
Schedule
"""
strategies = [
vector.VectorMultiReductionLowering.InnerReduction,
vector.VectorMultiReductionLowering.InnerParallel,
]
with schedule_boilerplate() as (schedule, named_seq):
# First lower reductions already matching a strategy.
with ir.InsertionPoint(
transform.ApplyPatternsOp(named_seq.bodyTarget).patterns
):
for strategy in strategies:
vector.apply_patterns_vector_multi_reduction_flattening(
lowering_strategy=strategy
)
vector.apply_patterns_vector_multi_reduction_unrolling(
lowering_strategy=strategy
)
# Reordering transposes every mismatched reduction, so run it on
# the leftovers only, with one strategy only as using both strategies
# creates conflicts.
fallback = vector.VectorMultiReductionLowering.InnerParallel
with ir.InsertionPoint(
transform.ApplyPatternsOp(named_seq.bodyTarget).patterns
):
vector.apply_patterns_vector_reorder_multi_reduction_dims(
lowering_strategy=fallback
)
vector.apply_patterns_vector_multi_reduction_flattening(
lowering_strategy=fallback
)
vector.apply_patterns_vector_multi_reduction_unrolling(
lowering_strategy=fallback
)
lh_transform.cleanup(named_seq.bodyTarget)
transform.yield_()
return schedule


def x86_vectorization() -> ir.Module:
"""
Apply x86-specific vector rewrites.
Expand Down
80 changes: 80 additions & 0 deletions test/transform/test_lower_multi_reductions.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
# RUN: %PYTHON %s | FileCheck %s

from mlir import ir

import lighthouse.dialects as lh_dialects
from lighthouse.schedule import lower_vector_multi_reductions


def run(name: str, payload_str: str):
print(f"Test: {name}", flush=True)
with ir.Context(), ir.Location.unknown():
lh_dialects.register_and_load()
payload = ir.Module.parse(payload_str)
sched = lower_vector_multi_reductions()
sched.body.operations[0].apply(payload.operation)
print(payload)


# Inner (row) reduction: one horizontal reduction per row, no transpose.
# CHECK-LABEL: Test: row
# CHECK-NOT: vector.transpose
# CHECK-COUNT-2: vector.reduction <add>, %{{.*}}, %{{.*}} : vector<64xf32> into f32
# CHECK-NOT: vector.multi_reduction
run(
"row",
"""
func.func @row(%a: vector<2x64xf32>, %acc: vector<2xf32>) -> vector<2xf32> {
%0 = vector.multi_reduction <add>, %a, %acc [1] : vector<2x64xf32> to vector<2xf32>
return %0 : vector<2xf32>
}
""",
)


# Outer (column) reduction: lane-wise adds, no transpose.
# CHECK-LABEL: Test: column
# CHECK-NOT: vector.transpose
# CHECK-COUNT-2: arith.addf %{{.*}}, %{{.*}} : vector<32xf32>
# CHECK-NOT: vector.multi_reduction
run(
"column",
"""
func.func @column(%a: vector<2x32xf32>, %acc: vector<32xf32>) -> vector<32xf32> {
%0 = vector.multi_reduction <add>, %a, %acc [0] : vector<2x32xf32> to vector<32xf32>
return %0 : vector<32xf32>
}
""",
)


# 1-D reduction lowers to a single horizontal reduction.
# CHECK-LABEL: Test: rank1
# CHECK: vector.reduction <maximumf>, %{{.*}}, %{{.*}} : vector<128xf32> into f32
# CHECK-NOT: vector.multi_reduction
run(
"rank1",
"""
func.func @rank1(%a: vector<128xf32>, %acc: f32) -> f32 {
%0 = vector.multi_reduction <maximumf>, %a, %acc [0] : vector<128xf32> to f32
return %0 : f32
}
""",
)


# A reduced middle dim is not handled by either strategy directly and falls
# back to reordering it outward.
# CHECK-LABEL: Test: middle
# CHECK: vector.transpose
# CHECK-NOT: vector.multi_reduction
# CHECK: return
run(
"middle",
"""
func.func @middle(%a: vector<2x4x16xf32>, %acc: vector<2x16xf32>) -> vector<2x16xf32> {
%0 = vector.multi_reduction <add>, %a, %acc [1] : vector<2x4x16xf32> to vector<2x16xf32>
return %0 : vector<2x16xf32>
}
""",
)
Loading