2020 reduction_dims ,
2121)
2222from lighthouse .dialects .transform .transform_ext .utils .ir_rewrite import (
23+ depends_on_op ,
2324 operand_eliminating_constant ,
2425)
2526
@@ -266,6 +267,66 @@ def find_r2_elementwise_operand(r2: ir.OpView, e: ir.OpView) -> int:
266267 return found
267268
268269
270+ def _check_r2_inputs (
271+ r1_loop : ir .OpView ,
272+ e : ir .OpView ,
273+ r2 : ir .OpView ,
274+ e_input_index : int ,
275+ ) -> None :
276+ """Require one E input, optionally with an independent second input."""
277+ inputs = linalg_inputs (r2 )
278+ if len (inputs ) not in (1 , 2 ):
279+ raise FusionRejected ("R2 must have one or two inputs" )
280+ if len (inputs ) == 2 :
281+ other = inputs [1 - e_input_index ]
282+ if depends_on_op (other , e ) or depends_on_op (other , r1_loop ):
283+ raise FusionRejected (
284+ "R2 contribution has another input dependent on E or R1"
285+ )
286+
287+
288+ def _check_r2_body_pattern (
289+ r2 : ir .OpView , e_input_index : int , contribution : ir .Value , combiner : ir .Operation
290+ ) -> None :
291+ """Accept casts of inputs, then E + acc or E * V + acc."""
292+ body = r2 .regions [0 ].blocks [0 ]
293+ ops = [opview (op ) for op in list (body .operations )[:- 1 ]]
294+ first_core = 0
295+ # Skip casting operations at the beginning of the R2 body
296+ while first_core < len (ops ) and isinstance (
297+ ops [first_core ], (arith .ExtFOp , arith .TruncFOp )
298+ ):
299+ first_core += 1
300+ core = ops [first_core :]
301+
302+ def input_arg (value : ir .Value ) -> ir .Value :
303+ """Follow input casts back to a scalar R2 block argument."""
304+ producer = defining_op (value )
305+ while (
306+ producer is not None
307+ and producer .block == body
308+ and isinstance (producer .opview , (arith .ExtFOp , arith .TruncFOp ))
309+ ):
310+ value = producer .operands [0 ]
311+ producer = defining_op (value )
312+ return value
313+
314+ if not core or core [- 1 ].operation != combiner :
315+ raise FusionRejected ("R2 body must end in its accumulator addf" )
316+ if len (linalg_inputs (r2 )) == 1 :
317+ if len (core ) == 1 and input_arg (contribution ) == body .arguments [e_input_index ]:
318+ return
319+ elif len (core ) == 2 and isinstance (core [0 ], arith .MulFOp ):
320+ mul = core [0 ]
321+ factors = [input_arg (value ) for value in mul .operands ]
322+ input_args = set (list (body .arguments )[:2 ])
323+ if contribution == mul .results [0 ] and set (factors ) == input_args :
324+ return
325+ raise FusionRejected (
326+ "R2 body must add E or multiply two distinct inputs before adding"
327+ )
328+
329+
269330def needs_elementwise_clone (e : ir .OpView , r2 : ir .OpView ) -> bool :
270331 """Preserve E outside the loop when another user needs its final result."""
271332 r2_op = r2 .operation
@@ -547,7 +608,8 @@ def check_legal_fusion_triple(
547608 Each non-sibling input also feeds E after tracing tile slices. Their
548609 equal-rank, dimension-only maps yield a consistent, complete R1-to-E
549610 loop mapping that aligns the reduction axes.
550- * R2's body is a single ``arith.addf`` reduction with a zero init (matmul contractions are allowed).
611+ * R2 sums E or a product of E and one independent input. Input casts may
612+ precede the final ``mulf``/``addf``; its init is zero.
551613 * R2's result type is f16, bf16, f32, or f64, and E's result type can be
552614 widened with it to evaluate the correction.
553615 * E's scalar body can be cloned at that type: captured values can convert
@@ -580,6 +642,7 @@ def check_legal_fusion_triple(
580642 raise FusionRejected ("R1 loop, E and R2 are not all in the same block" )
581643
582644 r2_e_operand = find_r2_elementwise_operand (r2 , e )
645+ _check_r2_inputs (r1_loop , e , r2 , r2_e_operand )
583646
584647 r2_red_dims = reduction_dims (r2 )
585648 if len (r2_red_dims ) != 1 :
@@ -705,7 +768,10 @@ def check_legal_fusion_triple(
705768 )
706769 check_inner_reduction_against_elementwise (inner , e , e_tiled_dim , inner_results )
707770
708- _ , combiners = _match_reduction ([r2 .regions [0 ].blocks [0 ].arguments [- 1 ]], 0 )
771+ # `contribution` refers to the part of R2 that gets added to its accumulator.
772+ contribution , combiners = _match_reduction (
773+ [r2 .regions [0 ].blocks [0 ].arguments [- 1 ]], 0
774+ )
709775 if not combiners :
710776 raise FusionRejected ("R2's region does not match a reduction pattern" )
711777 if len (combiners ) != 1 :
@@ -714,6 +780,7 @@ def check_legal_fusion_triple(
714780 )
715781 if not isinstance (combiners [0 ].opview , arith .AddFOp ):
716782 raise FusionRejected (f"R2's combiner is not arith.addf: { combiners [0 ].name } " )
783+ _check_r2_body_pattern (r2 , r2_e_operand , contribution , combiners [0 ])
717784
718785 r2_init = linalg_outputs (r2 )[0 ]
719786 if not _is_defined_as_zero (r2_init ):
0 commit comments