Skip to content

Commit 26cafe2

Browse files
committed
cleanup
1 parent 99a1e00 commit 26cafe2

7 files changed

Lines changed: 59 additions & 72 deletions

File tree

‎examples/xegpu/nanoGPT_schedule.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -259,7 +259,7 @@ def _fuse_attention_in_region(anytype, forall, fa_params):
259259
static_sizes=[0, 0, tile_size],
260260
scalable_sizes=[False, False, False],
261261
)
262-
transform.annotate(reduction_loop, "__reduction_loop__")
262+
transform.annotate(reduction_loop, transform_ext.REDUCTION_LOOP_ATTR_NAME)
263263

264264
# First chain: max -> p -> row sum. `p` also feeds the contraction, so the op
265265
# fuses a clone of it and leaves the original in place for the second chain.

‎lighthouse/dialects/transform/transform_ext/__init__.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,10 @@
2626
from .ops.clear_tile_and_fuse_annotations import clear_tile_and_fuse_annotations
2727
from .ops.get_fusion_roots import get_fusion_roots
2828
from .ops.propagate_tile_sizes import propagate_tile_sizes
29+
from .utils.dependant_reduction_legality import REDUCTION_LOOP_ATTR_NAME
2930

3031
__all__ = [
32+
"REDUCTION_LOOP_ATTR_NAME",
3133
"TransformExtensionDialect",
3234
"assign_tile_sizes",
3335
"clear_tile_and_fuse_annotations",

‎lighthouse/dialects/transform/transform_ext/ops/enable_fastmath_optimizations.py‎

Lines changed: 13 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -68,13 +68,17 @@ def _annotate_fastmath(root: ir.Operation) -> None:
6868
def _fastmath_allows_transform(op: ir.Operation) -> bool:
6969
"""True if `op`'s `fastmath` flags permit `exp(a)/exp(b)` -> `exp(a-b)`.
7070
71-
The rewrite is not IEEE-preserving: it changes the rounding of the two
72-
intermediate exponentials into a single one, and it turns a division into a
73-
subtraction. `reassoc` is what licenses that regrouping, and the flag set also
74-
has to cover the values the identity ignores -- `exp(a)/exp(b)` is finite when
75-
both exponentials overflow or underflow together, whereas `exp(a-b)` is not
76-
(and vice versa), so `nnan`/`ninf` are required too. `fastmath<fast>` is the
77-
only flag set that provides all three.
71+
The rewrite is not IEEE-preserving and relies on three flags:
72+
73+
- `reassoc`, which licenses regrouping the expression at all.
74+
- `nnan` and `ninf`, because the two forms disagree once an exponential
75+
overflows or underflows. For `a = b = 1000`, `exp(a)/exp(b)` is `inf/inf`,
76+
i.e. NaN, while `exp(a-b)` is 1.
77+
78+
Only the full `fastmath<fast>` set is accepted. That is stricter than the
79+
three flags above require -- `<reassoc,nnan,ninf>` would also be sound -- but
80+
it is the only set the payloads here produce, so the check stays a plain
81+
comparison.
7882
"""
7983
if "fastmath" not in op.attributes:
8084
return False
@@ -134,21 +138,13 @@ class EnableFastmathOptimizationsOp(
134138
2. Rewrite `exp(a) / exp(b)` into `exp(a - b)`, trading a transcendental and
135139
a divide for a subtract. Only applied where the `arith.divf` and both
136140
`math.exp` producers carry `fastmath<fast>`; see
137-
`_fastmath_allows_transform` for why nothing weaker is accepted.
141+
`_fastmath_allows_transform` for which flags the rewrite relies on.
138142
139143
The order matters and is why the two are one op: the fold keys off the
140144
annotation. Ops without the flag are skipped rather than rejected, so this is
141145
safe to run over a whole function.
142146
143-
The motivating case is the online-softmax rescale factor emitted by
144-
`fuse_dependant_reduction_ops`, which is `exp(-m_new) / exp(-m_old)`; this turns
145-
it into the single `exp(m_old - m_new)` a hand-written flash-attention kernel
146-
uses. That factor is built from `linalg.elementwise`, which has no `fastmath`
147-
attribute, so the `arith.divf` it expands to arrives unannotated even when the
148-
surrounding payload ops are `fastmath<fast>` -- hence step 1.
149-
150-
Run after vectorization: before it the two exponentials and the divide live in
151-
three separate `linalg.generic`s, so there is no single op to match.
147+
Ideally must be run after vectorization.
152148
153149
Args:
154150
target: Handle to root ops to work within (e.g. func.func).

‎lighthouse/dialects/transform/transform_ext/ops/fuse_dependant_reduction_ops.py‎

Lines changed: 15 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -30,47 +30,42 @@ class FuseDependantReductionOpsOp(
3030
TransformExtensionDialect.Operation, name="fuse_dependant_reduction_ops"
3131
):
3232
"""
33-
Fuse a dependent elementwise + consumer reduction into an already-tiled
34-
producer reduction loop.
35-
3633
Fuses a dependency chain ``R1 -> E -> R2`` into a single online (one-pass)
3734
reduction loop, where ``R1`` is the producer reduction (`tiled_reduction_loop`),
3835
``E`` is an elementwise op (`elementwise_op`) consuming ``R1``'s result as a
3936
broadcast input, and ``R2`` is the consumer reduction (`reduction_op`) reducing
40-
``E``'s output over the same axis. The canonical case is two-pass softmax
41-
becoming the online form; an attention chain is the same shape with a `P @ V`
42-
contraction as a second ``R2``.
37+
``E``'s output over the same axis.
4338
4439
``E`` is kept as a *separate* elementwise op rather than folded into ``R2``'s
4540
body; this keeps the per-element term explicit and easier to vectorize later.
4641
Because ``E`` is exactly the per-element term that ``R2`` reduces, the online
47-
correction factor is derived directly from ``E`` (evaluate ``E`` with its data
48-
inputs set to the neutral element and the consumed accumulators bound to their
49-
new/old values), without inspecting ``R2``'s body.
42+
correction factor is derived directly from ``E``.
5043
5144
The producer must already be tiled along the shared reduction dimension into
5245
an ``scf.for`` annotated with the ``__reduction_loop__`` unit attribute; the
5346
tile size is read from that loop's step. This op does not tile the producer
5447
itself -- use ``transform.structured.tile_using_for`` followed by
5548
``transform.annotate`` for that.
49+
TODO: Remove the annotation requirement for the producer reduction loop.
5650
57-
``E`` may feed consumers besides ``R2``. If it does, this op fuses a *clone* of
58-
``E`` and leaves the original in place for them. The fused copy computes each
59-
tile against the *running* accumulator, which the online correction only
51+
This op fuses a *clone* of
52+
``E`` and leaves the original in place for them if it has other users. The fused copy computes each
53+
tile against the *running* `R1` accumulator, which the online correction only
6054
accounts for on ``R2``'s accumulator, so ``E``'s own full-extent result is
61-
stale in every tile but the last and must not be read off the loop -- softmax's
62-
normalizing divide, for one, needs ``E`` evaluated at the *final* max. The
55+
stale in every tile but the last and must not be read off the loop. The
6356
original ``E`` still reads ``R1``'s final result off the loop, so it recomputes
6457
the term correctly; the clone's full-extent loop result is dead, and the
6558
schedule drops it with ``remove-dead-values`` after vectorization.
6659
6760
A consumer that *reduces* ``E`` over the same axis is a candidate ``R2`` in its
68-
own right, and is fused into the same loop by applying this op again -- the
69-
clone is what leaves it a term to fuse. Applying the op once per consumer
70-
reduction therefore fuses an attention chain (one ``exp`` term feeding both a
71-
row sum and a `P @ V` contraction) into a single loop. The returned loop keeps
72-
the ``__reduction_loop__`` attribute, so no re-annotation is needed between
73-
applications.
61+
own right, and is fused into the same loop by this op along with necessary
62+
*online correction*. Applying the op once per consumer reduction therefore
63+
fuses an attention chain (one ``exp`` term feeding both a row sum and a `P @ V`
64+
contraction) into a single loop. The returned loop keeps the ``__reduction_loop__``
65+
attribute, so no re-annotation is needed between applications.
66+
67+
For more details about the fusion algorithm refer:
68+
https://discourse.llvm.org/t/linalg-rfc-an-approach-for-tiling-and-fusing-dependent-reductions-in-linalg/91698
7469
7570
Return modes:
7671
Each handle must point to exactly one payload op, otherwise this produces

‎lighthouse/dialects/transform/transform_ext/utils/dependant_reduction_legality.py‎

Lines changed: 25 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,6 @@
2727
__all__ = [
2828
"REDUCTION_LOOP_ATTR_NAME",
2929
"FusionRejected",
30-
"check_elementwise_separability",
3130
"check_legal_fusion_triple",
3231
"collect_inner_reduction_generics",
3332
"collect_r1_as_elementwise_inputs",
@@ -136,29 +135,17 @@ def find_r2_elementwise_operand(r2: ir.OpView, e: ir.OpView) -> ls.Operand:
136135
def needs_elementwise_clone(e: ir.OpView, r2: ir.OpView) -> bool:
137136
"""Whether the fusion must work on a *clone* of `e` rather than `e` itself.
138137
139-
Fusing ``E`` moves it into the loop, where each tile is computed against the
140-
*running* value of ``R1``'s accumulator rather than its final one. That is
141-
exactly what the online correction accounts for -- but only for ``R2``'s
142-
accumulator. ``E``'s own full-extent result is therefore stale in every tile
143-
but the last, so no consumer outside the loop may read it. For softmax
144-
``E = exp(x - m)``, the tile written at step ``t`` is ``exp(x - m_t)`` and
145-
differs from the final ``exp(x - m)`` by ``exp(m - m_t)``; a normalizing
146-
divide reading it off the loop would normalize stale numerators and get row
147-
sums above one.
148-
149-
So whenever ``E`` has another consumer, fuse a clone and leave the original
150-
in place, where it still reads ``R1``'s final result off the loop and
151-
recomputes the term correctly. That covers both shapes this arises in: an
152-
all-parallel consumer (softmax's normalizing divide), and another consumer
153-
*reduction* over the same axis (an attention chain's row sum alongside its
154-
``P @ V`` contraction), which is a candidate ``R2`` in its own right and gets
155-
fused into the same loop by applying the fusion again.
138+
True if ``E`` has any user besides ``R2``. Fusing ``E`` computes each tile
139+
against the *running* ``R1`` accumulator instead of its final result, so
140+
``E``'s own output is stale in every tile but the last and no user outside the
141+
loop may read it. The original is left in place for those users, where it still
142+
reads ``R1``'s final result and recomputes the term correctly.
156143
"""
157144
r2_op = r2.operation
158145
return any(user != r2_op for user in op_users(e.results[0]))
159146

160147

161-
# --- separability ------------------------------------------------------------
148+
# --- separability analysis ------------------------------------------------------------
162149

163150
#: `v` is independent of every block argument (a loop-invariant scalar).
164151
_CONST = 1 << 0
@@ -356,12 +343,22 @@ def check_inner_reduction_against_elementwise(
356343
357344
``r1`` must reduce over exactly one innermost loop, and every ``r1`` input
358345
(resolved *through* its tile ``extract_slice`` to the source tensor) must also
359-
appear as an ``e`` input -- *except* inputs that are results of a sibling
360-
inner reduction (`inner_results`), which are themselves broadcast running
361-
accumulators. The shared inputs' indexing maps must agree under a consistent
362-
injective dim-mapping phi from ``r1``'s loop dims to ``e``'s loop dims, with
363-
phi total over ``r1``'s loops and aligning ``r1``'s reduction dim with the
364-
``e`` dim carrying ``R2``'s reduction axis (`e_tiled_dim`).
346+
appear as an ``e`` input.
347+
348+
Fusion rewrites ``r1``'s indexing maps into ``e``'s iteration space, so the two
349+
ops' loop dims have to correspond. That correspondence ``phi`` is not given but
350+
*derived*, by unifying the maps of the inputs they share: at each position of a
351+
shared input's map, the ``r1`` dim indexing that tensor dim pairs with the ``e``
352+
dim indexing it, and the pairing must be consistent across all shared inputs.
353+
For softmax's ``max`` and ``exp(x - m)``, both read ``x`` as
354+
``(d0, d1) -> (d0, d1)``, so ``phi = {0: 0, 1: 1}``.
355+
356+
``phi`` must additionally be
357+
358+
* total over ``r1``'s loop dims, so every ``r1`` map -- its init map included
359+
-- can be translated, and
360+
* reduction-aligned: ``phi[r1_red_dim] == e_tiled_dim``, i.e. ``r1`` reduces
361+
the same axis that ``R2`` reduces through ``e``.
365362
366363
Note the comparison is against ``E``, not ``R2``: with the elementwise term
367364
left unfused, ``E`` is the op sharing ``R1``'s data inputs (e.g. ``x`` in
@@ -377,8 +374,8 @@ def check_inner_reduction_against_elementwise(
377374
if r1_red_dims[0] != ls.num_loops(r1) - 1:
378375
raise FusionRejected("reduction iterator is not the innermost loop in inner R1")
379376

380-
# Every input of R1 must also appear as an input of E, and the two ops' maps
381-
# for each shared input must agree up to a consistent injective mapping phi.
377+
# Every input of R1 must also appear as an input of E. Unifying the two ops'
378+
# maps for each shared input derives phi: R1 loop dim -> E loop dim.
382379
phi: dict[int, int] = {}
383380

384381
def try_add_mapping(r1_dim: int, e_dim: int) -> bool:
@@ -460,6 +457,7 @@ def check_legal_fusion_triple(
460457
reduction axis, which the retiling step needs, and the tile size.
461458
462459
Raises `FusionRejected` with the reason on any violated condition.
460+
TODO: These conditions are too strict, reconsider and relax later.
463461
"""
464462
if len(list(r2.results)) != 1 or ls.num_dps_inits(r2) != 1:
465463
raise FusionRejected(

‎lighthouse/schedule/xegpu/fused_attention_schedule.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -175,7 +175,7 @@ def _derive_flash_attention(anytype, func, layer_params):
175175
static_sizes=[0, 0, reduction_tile],
176176
scalable_sizes=[False, False, False],
177177
)
178-
transform.annotate(reduction_loop, "__reduction_loop__")
178+
transform.annotate(reduction_loop, transform_ext.REDUCTION_LOOP_ATTR_NAME)
179179

180180
# First chain: max -> p -> row sum. `p` also feeds the contraction, so the op
181181
# fuses a clone of it and leaves the original in place for the second chain.

‎test/transform/test_fuse_dependant_reduction_ops.py‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -23,10 +23,6 @@
2323
from lighthouse.dialects.transform import transform_ext
2424
from lighthouse.schedule.builders import schedule_boilerplate
2525

26-
#: Marks the tiled producer loop. A plain `scf.for` carries no iterator-type
27-
#: metadata, so this is how the fusion recognises a tiled reduction axis.
28-
REDUCTION_LOOP_ATTR = "__reduction_loop__"
29-
3026

3127
# ---------------------------------------------------------------------------
3228
# Payloads
@@ -184,7 +180,7 @@ def online_softmax_schedule(tile_size: int = 32) -> ir.Module:
184180
# Tile R1 along the reduction dim only, giving the `scf.for` the fusion
185181
# needs, and mark it as a reduction loop.
186182
_tiled_r1, r1_loop = structured.TileUsingForOp(r1, sizes=[0, tile_size]).results
187-
transform.annotate(r1_loop, REDUCTION_LOOP_ATTR)
183+
transform.annotate(r1_loop, transform_ext.REDUCTION_LOOP_ATTR_NAME)
188184

189185
fused = transform_ext.fuse_dependant_reduction_ops(e, r2, r1_loop)
190186
transform.annotate(fused, "online_softmax_loop")
@@ -202,7 +198,7 @@ def flash_attention_schedule(tile_size: int = 32) -> ir.Module:
202198
r1, e, r2a, r2b, _divide = transform.split_handle([anyop] * 5, generics)
203199

204200
_tiled_r1, r1_loop = structured.TileUsingForOp(r1, sizes=[0, tile_size]).results
205-
transform.annotate(r1_loop, REDUCTION_LOOP_ATTR)
201+
transform.annotate(r1_loop, transform_ext.REDUCTION_LOOP_ATTR_NAME)
206202

207203
# First chain: the row sum. This fuses a *clone* of E, since E still feeds
208204
# the contraction, and leaves the original E in place for it.

0 commit comments

Comments
 (0)