@@ -20,12 +20,15 @@ class SinkNormalizationPastContractionOp(
2020
2121 sum_k (A[k] / S) * B[k] == (sum_k A[k] * B[k]) / S
2222
23- Worth doing when the reduction dim very large compared to the other dims.
23+ Worth doing when the reduction dim is very large compared to the other dims.
24+
25+ The scale is expected inside the contraction's body, i.e. scale is already
26+ fused into the contraction's body.
2427
2528 Before:
2629 ```
27- %p = linalg.generic ins(%e, %l) { arith.divf } (all-parallel )
28- %o = linalg.generic ins(%p, %v) { mulf, addf } (contraction over k)
30+ %o = linalg.generic ins(%e, %l, %v ) { (contraction over k )
31+ divf, mulf, addf }
2932 ```
3033
3134 After:
@@ -34,36 +37,29 @@ class SinkNormalizationPastContractionOp(
3437 %n = linalg.generic ins(%o, %l) { arith.divf } (all-parallel)
3538 ```
3639
37- Both payload ops are named explicitly. The op checks that the sink applies and
38- reports a silenceable error saying why if it does not. It requires:
39-
40- * Exactly one payload op per handle;
41- * `contraction` to be a structured linalg op (i.e. a `linalg.generic` or a named
42- op such as ``linalg.matmul``/``linalg.batch_matmul``) with exactly one
43- reduction dim, reduction dim is the innermost and, has an identity output map;
44- * `normalization` to be an all-parallel two-input `linalg.generic` whose body
45- is a single ``arith.divf``/``arith.mulf`` of two block arguments, with an
46- identity output map with contraction as its only user;
47- * The scale, seen from the contraction's iteration space, not to reference the
48- reduction dim -- the condition that lets it factor out;
49- * The scale's element type to be convertible to the contraction's accumulator
40+ Op checks that the
41+ sink applies and reports a silenceable error saying why if it does not. It
42+ requires:
43+
44+ * Exactly one payload op for `contraction`;
45+ * `contraction` to be a `linalg.generic` with exactly one reduction dim, that
46+ dim innermost, and an identity output map;
47+ * its body to hold an ``arith.divf``/``arith.mulf`` on two input block
48+ arguments;
49+ * the scale operand not to reference the reduction dim -- the condition that
50+ lets it factor out;
51+ * the scale's element type to be convertible to the contraction's accumulator
5052 type (it is widened when the divide moves).
5153
52- A named contraction additionally has to be able to read the numerator under the
53- indexing map already in place, since its own verifier constrains those maps;
54- generalize it to a `linalg.generic` first if not.
55-
56- `contraction` is rewritten in place, so a handle to it stays valid.
57-
5854 Args:
59- normalization: Handle to the scaling linalg.generic.
60- contraction: Handle to the contraction consuming its result.
55+ contraction: Handle to the contraction carrying the scale.
6156 Returns:
62- Handle to the new linalg.generic applying the scale after the contraction.
57+ rewritten_contraction: The contraction rebuilt without the scale operand.
58+ sunk_normalization: The new linalg.generic applying the scale after it.
6359 """
6460
65- normalization : ext .Operand [transform .AnyOpType ]
6661 contraction : ext .Operand [transform .AnyOpType ]
62+ rewritten_contraction : ext .Result [transform .AnyOpType [()]] = ext .infer_result ()
6763 sunk_normalization : ext .Result [transform .AnyOpType [()]] = ext .infer_result ()
6864
6965 @classmethod
@@ -85,23 +81,18 @@ def reject(message: str) -> DiagnosedSilenceableFailure:
8581 )
8682 return DiagnosedSilenceableFailure .SilenceableFailure
8783
88- payloads = []
89- for name , handle in (
90- ("normalization" , op .normalization ),
91- ("contraction" , op .contraction ),
92- ):
93- handle_ops = state .get_payload_ops (handle )
94- if len (handle_ops ) != 1 :
95- return reject (
96- f"expected exactly one payload op for '{ name } ', got "
97- f"{ len (handle_ops )} "
98- )
99- payloads .append (handle_ops [0 ])
100-
101- sunk , error = _sink_norm (payloads [0 ], payloads [1 ], rewriter )
84+ targets = state .get_payload_ops (op .contraction )
85+ if len (targets ) != 1 :
86+ return reject (
87+ f"expected exactly one payload op for 'contraction', got "
88+ f"{ len (targets )} "
89+ )
90+
91+ result , error = _sink_norm (targets [0 ], rewriter )
10292 if error is not None :
10393 return reject (error )
104- results .set_ops (op .sunk_normalization , [sunk .operation ])
94+ results .set_ops (op .rewritten_contraction , [result .contraction .operation ])
95+ results .set_ops (op .sunk_normalization , [result .normalization .operation ])
10596 return DiagnosedSilenceableFailure .Success
10697
10798 @staticmethod
@@ -113,23 +104,18 @@ def allow_repeated_handle_operands(
113104 class MemoryEffectsOpInterfaceModel (ir .MemoryEffectsOpInterface ):
114105 @staticmethod
115106 def get_effects (op : ir .Operation ):
116- # The normalization is erased, so its handle is consumed; the
117- # contraction is rewritten in place, so its handle survives.
118- operands = list (op .op_operands )
107+ # The in-body form rebuilds the contraction and erases the original, so
108+ # the handle is consumed; `rewritten_contraction` replaces it.
119109 return (
120- transform .consumes_handle (operands [:1 ])
121- + transform .only_reads_handle (operands [1 :])
110+ transform .consumes_handle (op .op_operands )
122111 + transform .produces_handle (op .results )
123112 + transform .modifies_payload ()
124113 )
125114
126115
127116def sink_normalization_past_contraction (
128- normalization : ir .Value [transform .AnyOpType ],
129117 contraction : ir .Value [transform .AnyOpType ],
130- ) -> ir .Value [transform .AnyOpType ]:
118+ ) -> tuple [ ir .Value [transform .AnyOpType ], ir . Value [ transform . AnyOpType ] ]:
131119 """snake_case wrapper to create SinkNormalizationPastContractionOp."""
132- op = SinkNormalizationPastContractionOp (
133- normalization = normalization , contraction = contraction
134- )
135- return op .sunk_normalization
120+ op = SinkNormalizationPastContractionOp (contraction = contraction )
121+ return op .rewritten_contraction , op .sunk_normalization
0 commit comments