2020 reduction_dims ,
2121)
2222from lighthouse .dialects .transform .transform_ext .utils .ir_rewrite import (
23+ depends_on_op ,
2324 operand_eliminating_constant ,
2425)
2526
@@ -266,6 +267,66 @@ def find_r2_elementwise_operand(r2: ir.OpView, e: ir.OpView) -> int:
266267 return found
267268
268269
270+ def _check_r2_inputs (
271+ r1_loop : ir .OpView ,
272+ e : ir .OpView ,
273+ r2 : ir .OpView ,
274+ e_input_index : int ,
275+ ) -> None :
276+ """Require one E input, optionally with an independent second input."""
277+ inputs = linalg_inputs (r2 )
278+ if len (inputs ) not in (1 , 2 ):
279+ raise FusionRejected ("R2 must have one or two inputs" )
280+ if len (inputs ) == 2 :
281+ other = inputs [1 - e_input_index ]
282+ if depends_on_op (other , e ) or depends_on_op (other , r1_loop ):
283+ raise FusionRejected (
284+ "R2 contribution has another input dependent on E or R1"
285+ )
286+
287+
288+ def _check_r2_body_pattern (
289+ r2 : ir .OpView , e_input_index : int , contribution : ir .Value , combiner : ir .Operation
290+ ) -> None :
291+ """Accept casts of inputs, then E + acc or E * V + acc."""
292+ body = r2 .regions [0 ].blocks [0 ]
293+ ops = [opview (op ) for op in list (body .operations )[:- 1 ]]
294+ first_core = 0
295+ # Skip casting operations at the beginning of the R2 body
296+ while first_core < len (ops ) and isinstance (
297+ ops [first_core ], (arith .ExtFOp , arith .TruncFOp )
298+ ):
299+ first_core += 1
300+ core = ops [first_core :]
301+
302+ def input_arg (value : ir .Value ) -> ir .Value :
303+ """Follow input casts back to a scalar R2 block argument."""
304+ producer = defining_op (value )
305+ while (
306+ producer is not None
307+ and producer .block == body
308+ and isinstance (producer .opview , (arith .ExtFOp , arith .TruncFOp ))
309+ ):
310+ value = producer .operands [0 ]
311+ producer = defining_op (value )
312+ return value
313+
314+ if not core or core [- 1 ].operation != combiner :
315+ raise FusionRejected ("R2 body must end in its accumulator addf" )
316+ if len (linalg_inputs (r2 )) == 1 :
317+ if len (core ) == 1 and input_arg (contribution ) == body .arguments [e_input_index ]:
318+ return
319+ elif len (core ) == 2 and isinstance (core [0 ], arith .MulFOp ):
320+ mul = core [0 ]
321+ factors = [input_arg (value ) for value in mul .operands ]
322+ input_args = set (list (body .arguments )[:2 ])
323+ if contribution == mul .results [0 ] and set (factors ) == input_args :
324+ return
325+ raise FusionRejected (
326+ "R2 body must add E or multiply two distinct inputs before adding"
327+ )
328+
329+
269330def needs_elementwise_clone (e : ir .OpView , r2 : ir .OpView ) -> bool :
270331 """Preserve E outside the loop when another user needs its final result."""
271332 r2_op = r2 .operation
@@ -492,11 +553,15 @@ def try_add_mapping(r1_dim: int, e_dim: int) -> bool:
492553def _check_correction_body (
493554 e : ir .OpView , r1_input_indices : list [int ], compute_type : ir .Type
494555) -> None :
495- """Check if body of E's ops has valid data-operand stand-ins (safe values) ."""
556+ """Check E's constants and data-operand stand-ins before cloning its body ."""
496557 body = e .regions [0 ].blocks [0 ]
497558 accumulator_args = {body .arguments [index ] for index in r1_input_indices }
498559 for operation in list (body .operations )[:- 1 ]:
499560 op = opview (operation )
561+ if isinstance (op , arith .ConstantOp ) and not isinstance (
562+ op .operation .attributes ["value" ], ir .FloatAttr
563+ ):
564+ raise FusionRejected ("E body has a non-floating arith.constant" )
500565
501566 for operand in op .operands :
502567 if isinstance (operand , ir .BlockArgument ) and operand .owner == body :
@@ -524,10 +589,14 @@ def check_legal_fusion_triple(
524589 The checks require:
525590
526591 * E and R2 each have one result/init, and E has only parallel iterators.
592+ * All shaped operands and results of R1's loop and inner reductions, E,
593+ and R2 have static shapes, including the sources of R1 tile slices.
594+ TODO: Add support for dynamic shapes.
527595 * The R1 loop, E, and R2 occupy the same block.
528596 * Exactly one R2 input reads E's result.
529- * R2 has one innermost reduction iterator. Every R2 input map contains
530- that iterator and consists only of pure dimension expressions.
597+ * R2 has one innermost reduction iterator. Its output map preserves the
598+ parallel dimensions in order. Every R2 input map contains the reduction
599+ iterator and consists only of pure dimension expressions.
531600 * R2's map for E's result and E's output map have the same rank and only
532601 dimension expressions; together they identify the E reduction axis.
533602 * R1's loop bounds and step are constant, the step is positive, the R2
@@ -543,7 +612,8 @@ def check_legal_fusion_triple(
543612 Each non-sibling input also feeds E after tracing tile slices. Their
544613 equal-rank, dimension-only maps yield a consistent, complete R1-to-E
545614 loop mapping that aligns the reduction axes.
546- * R2's body is a single ``arith.addf`` reduction with a zero init (matmul contractions are allowed).
615+ * R2 sums E or a product of E and one independent input. Input casts may
616+ precede the final ``mulf``/``addf``; its init is zero.
547617 * R2's result type is f16, bf16, f32, or f64, and E's result type can be
548618 widened with it to evaluate the correction.
549619 * E's scalar body can be cloned at that type: captured values can convert
@@ -576,6 +646,7 @@ def check_legal_fusion_triple(
576646 raise FusionRejected ("R1 loop, E and R2 are not all in the same block" )
577647
578648 r2_e_operand = find_r2_elementwise_operand (r2 , e )
649+ _check_r2_inputs (r1_loop , e , r2 , r2_e_operand )
579650
580651 r2_red_dims = reduction_dims (r2 )
581652 if len (r2_red_dims ) != 1 :
@@ -585,6 +656,17 @@ def check_legal_fusion_triple(
585656 if r2_red_dims [0 ] != num_loops (r2 ) - 1 :
586657 raise FusionRejected ("reduction iterator is not the innermost loop in R2" )
587658
659+ r2_output_map = indexing_maps (r2 )[len (linalg_inputs (r2 ))]
660+ expected_output_map = ir .AffineMap .get (
661+ num_loops (r2 ),
662+ 0 ,
663+ [ir .AffineDimExpr .get (i ) for i in range (num_loops (r2 ) - 1 )],
664+ )
665+ if r2_output_map != expected_output_map :
666+ raise FusionRejected (
667+ f"R2 output map is not the identity projection: { r2_output_map } "
668+ )
669+
588670 # Each R2 input must admit slicing along the shared reduction axis.
589671 for index , _ in enumerate (linalg_inputs (r2 )):
590672 imap = indexing_maps (r2 )[index ]
@@ -632,6 +714,23 @@ def check_legal_fusion_triple(
632714 f"{ full_extent } "
633715 )
634716
717+ for name , op in [
718+ ("R1" , r1_loop ),
719+ * (("R1" , inner ) for inner in result_to_inner if inner is not None ),
720+ ("E" , e ),
721+ ("R2" , r2 ),
722+ ]:
723+ for value in (* op .operands , * op .results ):
724+ # Tiling gives an R1 input slice a static type even if its source
725+ # tensor is dynamic.
726+ values = (value , _resolve_slice_source (value )) if name == "R1" else (value ,)
727+ for shaped in values :
728+ if (
729+ isinstance (shaped .type , ir .ShapedType )
730+ and not shaped .type .has_static_shape
731+ ):
732+ raise FusionRejected (f"{ name } has a dynamic shape: { shaped .type } " )
733+
635734 e_red_range = _static_loop_ranges (e )[e_tiled_dim ]
636735 if ir .ShapedType .is_dynamic_size (e_red_range ) or e_red_range != full_extent :
637736 raise FusionRejected (
@@ -673,7 +772,10 @@ def check_legal_fusion_triple(
673772 )
674773 check_inner_reduction_against_elementwise (inner , e , e_tiled_dim , inner_results )
675774
676- _ , combiners = _match_reduction ([r2 .regions [0 ].blocks [0 ].arguments [- 1 ]], 0 )
775+ # `contribution` refers to the part of R2 that gets added to its accumulator.
776+ contribution , combiners = _match_reduction (
777+ [r2 .regions [0 ].blocks [0 ].arguments [- 1 ]], 0
778+ )
677779 if not combiners :
678780 raise FusionRejected ("R2's region does not match a reduction pattern" )
679781 if len (combiners ) != 1 :
@@ -682,6 +784,7 @@ def check_legal_fusion_triple(
682784 )
683785 if not isinstance (combiners [0 ].opview , arith .AddFOp ):
684786 raise FusionRejected (f"R2's combiner is not arith.addf: { combiners [0 ].name } " )
787+ _check_r2_body_pattern (r2 , r2_e_operand , contribution , combiners [0 ])
685788
686789 r2_init = linalg_outputs (r2 )[0 ]
687790 if not _is_defined_as_zero (r2_init ):
0 commit comments