Skip to content

Commit b9b054b

Browse files
committed
address comments
1 parent 7ac94c9 commit b9b054b

3 files changed

Lines changed: 153 additions & 2 deletions

File tree

‎lighthouse/dialects/transform/transform_ext/utils/dependent_reduction_fusion.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -234,6 +234,10 @@ def plan_correction_factor(
234234
and operand.owner == body
235235
and operand not in accumulator_args
236236
]
237+
# TODO: This assumes all the ops in E's body is data-dependent on the R1 results.
238+
# This is not true all the time, e.g., (x + x) * exp(x - m). Op (x + x)
239+
# does not depend on the R1 results, and therefore will be neutralized to
240+
# zero. Needs more analysis on E's body to correctly handle such cases.
237241
neutral = (
238242
irr.operand_eliminating_constant(op, compute_type) if data_args else None
239243
)
@@ -348,6 +352,9 @@ def build_term(pick) -> ir.Value:
348352
term_new = build_term(lambda acc: acc[1])
349353
term_old = build_term(lambda acc: acc[2])
350354

355+
# TODO: Evaluating two exp() values and dividing them may lead to numerical instability.
356+
# Rewriting the division into substraction require `reassoc` flags on exp. Explicitily check
357+
# these flags before doing the transformation.
351358
factor = _emit_elementwise(
352359
linalg.ElementwiseKind.div,
353360
lambda a, b: arith.divf(a, b),

‎lighthouse/dialects/transform/transform_ext/utils/dependent_reduction_legality.py‎

Lines changed: 69 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
reduction_dims,
2121
)
2222
from lighthouse.dialects.transform.transform_ext.utils.ir_rewrite import (
23+
depends_on_op,
2324
operand_eliminating_constant,
2425
)
2526

@@ -266,6 +267,66 @@ def find_r2_elementwise_operand(r2: ir.OpView, e: ir.OpView) -> int:
266267
return found
267268

268269

270+
def _check_r2_inputs(
271+
r1_loop: ir.OpView,
272+
e: ir.OpView,
273+
r2: ir.OpView,
274+
e_input_index: int,
275+
) -> None:
276+
"""Require one E input, optionally with an independent second input."""
277+
inputs = linalg_inputs(r2)
278+
if len(inputs) not in (1, 2):
279+
raise FusionRejected("R2 must have one or two inputs")
280+
if len(inputs) == 2:
281+
other = inputs[1 - e_input_index]
282+
if depends_on_op(other, e) or depends_on_op(other, r1_loop):
283+
raise FusionRejected(
284+
"R2 contribution has another input dependent on E or R1"
285+
)
286+
287+
288+
def _check_r2_body_pattern(
289+
r2: ir.OpView, e_input_index: int, contribution: ir.Value, combiner: ir.Operation
290+
) -> None:
291+
"""Accept casts of inputs, then E + acc or E * V + acc."""
292+
body = r2.regions[0].blocks[0]
293+
ops = [opview(op) for op in list(body.operations)[:-1]]
294+
first_core = 0
295+
# Skip casting operations at the beginning of the R2 body
296+
while first_core < len(ops) and isinstance(
297+
ops[first_core], (arith.ExtFOp, arith.TruncFOp)
298+
):
299+
first_core += 1
300+
core = ops[first_core:]
301+
302+
def input_arg(value: ir.Value) -> ir.Value:
303+
"""Follow input casts back to a scalar R2 block argument."""
304+
producer = defining_op(value)
305+
while (
306+
producer is not None
307+
and producer.block == body
308+
and isinstance(producer.opview, (arith.ExtFOp, arith.TruncFOp))
309+
):
310+
value = producer.operands[0]
311+
producer = defining_op(value)
312+
return value
313+
314+
if not core or core[-1].operation != combiner:
315+
raise FusionRejected("R2 body must end in its accumulator addf")
316+
if len(linalg_inputs(r2)) == 1:
317+
if len(core) == 1 and input_arg(contribution) == body.arguments[e_input_index]:
318+
return
319+
elif len(core) == 2 and isinstance(core[0], arith.MulFOp):
320+
mul = core[0]
321+
factors = [input_arg(value) for value in mul.operands]
322+
input_args = set(list(body.arguments)[:2])
323+
if contribution == mul.results[0] and set(factors) == input_args:
324+
return
325+
raise FusionRejected(
326+
"R2 body must add E or multiply two distinct inputs before adding"
327+
)
328+
329+
269330
def needs_elementwise_clone(e: ir.OpView, r2: ir.OpView) -> bool:
270331
"""Preserve E outside the loop when another user needs its final result."""
271332
r2_op = r2.operation
@@ -547,7 +608,8 @@ def check_legal_fusion_triple(
547608
Each non-sibling input also feeds E after tracing tile slices. Their
548609
equal-rank, dimension-only maps yield a consistent, complete R1-to-E
549610
loop mapping that aligns the reduction axes.
550-
* R2's body is a single ``arith.addf`` reduction with a zero init (matmul contractions are allowed).
611+
* R2 sums E or a product of E and one independent input. Input casts may
612+
precede the final ``mulf``/``addf``; its init is zero.
551613
* R2's result type is f16, bf16, f32, or f64, and E's result type can be
552614
widened with it to evaluate the correction.
553615
* E's scalar body can be cloned at that type: captured values can convert
@@ -580,6 +642,7 @@ def check_legal_fusion_triple(
580642
raise FusionRejected("R1 loop, E and R2 are not all in the same block")
581643

582644
r2_e_operand = find_r2_elementwise_operand(r2, e)
645+
_check_r2_inputs(r1_loop, e, r2, r2_e_operand)
583646

584647
r2_red_dims = reduction_dims(r2)
585648
if len(r2_red_dims) != 1:
@@ -705,7 +768,10 @@ def check_legal_fusion_triple(
705768
)
706769
check_inner_reduction_against_elementwise(inner, e, e_tiled_dim, inner_results)
707770

708-
_, combiners = _match_reduction([r2.regions[0].blocks[0].arguments[-1]], 0)
771+
# `contribution` refers to the part of R2 that gets added to its accumulator.
772+
contribution, combiners = _match_reduction(
773+
[r2.regions[0].blocks[0].arguments[-1]], 0
774+
)
709775
if not combiners:
710776
raise FusionRejected("R2's region does not match a reduction pattern")
711777
if len(combiners) != 1:
@@ -714,6 +780,7 @@ def check_legal_fusion_triple(
714780
)
715781
if not isinstance(combiners[0].opview, arith.AddFOp):
716782
raise FusionRejected(f"R2's combiner is not arith.addf: {combiners[0].name}")
783+
_check_r2_body_pattern(r2, r2_e_operand, contribution, combiners[0])
717784

718785
r2_init = linalg_outputs(r2)[0]
719786
if not _is_defined_as_zero(r2_init):

‎test/transform/test_dependent_reduction_legality.py‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -317,6 +317,83 @@ def main() -> None:
317317
# CHECK-NEXT: Legal to fuse: E dim 1, tile 32
318318
check_case("attention contraction", ATTENTION, r2_index=2)
319319

320+
# CHECK-LABEL: Case: casted attention contraction
321+
# CHECK-NEXT: Legal to fuse: E dim 1, tile 32
322+
check_case(
323+
"casted attention contraction",
324+
replace_once(
325+
ATTENTION,
326+
" %mul = arith.mulf %pv, %vv : f32",
327+
" %narrow = arith.truncf %pv : f32 to f16\n"
328+
" %wide = arith.extf %narrow : f16 to f32\n"
329+
" %mul = arith.mulf %wide, %vv : f32",
330+
),
331+
r2_index=2,
332+
)
333+
334+
# CHECK-LABEL: Case: squared E contribution
335+
# CHECK-NEXT: Not legal to fuse: R2 body must add E or multiply two distinct inputs before adding
336+
check_case(
337+
"squared E contribution",
338+
replace_once(
339+
SOFTMAX,
340+
" %a = arith.addf %in, %out : f32\n linalg.yield %a : f32",
341+
" %square = arith.mulf %in, %in : f32\n"
342+
" %a = arith.addf %square, %out : f32\n"
343+
" linalg.yield %a : f32",
344+
),
345+
)
346+
347+
# CHECK-LABEL: Case: narrowed E contribution
348+
# CHECK-NEXT: Legal to fuse: E dim 1, tile 32
349+
check_case(
350+
"narrowed E contribution",
351+
replace_once(
352+
SOFTMAX,
353+
" %a = arith.addf %in, %out : f32\n linalg.yield %a : f32",
354+
" %narrow = arith.truncf %in : f32 to f16\n"
355+
" %wide = arith.extf %narrow : f16 to f32\n"
356+
" %a = arith.addf %wide, %out : f32\n"
357+
" linalg.yield %a : f32",
358+
),
359+
)
360+
361+
e_dependent_factor = replace_once(
362+
SOFTMAX,
363+
" %s = linalg.generic {indexing_maps = [#rowcol, #row],",
364+
" %p_alias = tensor.cast %p : tensor<64x512xf32> to tensor<64x512xf32>\n"
365+
" %s = linalg.generic {indexing_maps = [#rowcol, #rowcol, #row],",
366+
)
367+
e_dependent_factor = replace_once(
368+
e_dependent_factor,
369+
"ins(%p : tensor<64x512xf32>) outs(%s_init",
370+
"ins(%p, %p_alias : tensor<64x512xf32>, tensor<64x512xf32>) outs(%s_init",
371+
)
372+
e_dependent_factor = replace_once(
373+
e_dependent_factor,
374+
" ^bb0(%in: f32, %out: f32):\n %a = arith.addf %in, %out : f32",
375+
" ^bb0(%in: f32, %alias: f32, %out: f32):\n"
376+
" %product = arith.mulf %in, %alias : f32\n"
377+
" %a = arith.addf %product, %out : f32",
378+
)
379+
# CHECK-LABEL: Case: E-dependent contraction factor
380+
# CHECK-NEXT: Not legal to fuse: R2 contribution has another input dependent on E or R1
381+
check_case("E-dependent contraction factor", e_dependent_factor)
382+
383+
duplicate_e_input = replace_once(
384+
e_dependent_factor,
385+
" %p_alias = tensor.cast %p : tensor<64x512xf32> to tensor<64x512xf32>\n",
386+
"",
387+
)
388+
duplicate_e_input = replace_once(
389+
duplicate_e_input,
390+
"ins(%p, %p_alias : tensor<64x512xf32>, tensor<64x512xf32>)",
391+
"ins(%p, %p : tensor<64x512xf32>, tensor<64x512xf32>)",
392+
)
393+
# CHECK-LABEL: Case: duplicate E contraction input
394+
# CHECK-NEXT: Not legal to fuse: R2 consumes E's result more than once
395+
check_case("duplicate E contraction input", duplicate_e_input)
396+
320397
transposed_r2 = ATTENTION.replace(
321398
"#ij = affine_map<(d0, d1, d2) -> (d0, d1)>",
322399
"#ij = affine_map<(d0, d1, d2) -> (d0, d1)>\n"

0 commit comments

Comments
 (0)