Skip to content

Commit d58cf9e

Browse files
committed
assume the scale is already fused into the contraction
1 parent 774ac96 commit d58cf9e

6 files changed

Lines changed: 361 additions & 539 deletions

File tree

‎lighthouse/dialects/transform/transform_ext/ops/replace_with_fused_attention.py‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -333,11 +333,9 @@ def apply(
333333
# f32 for numerical accuracy; only the matmul operands keep their
334334
# narrower element types.
335335
k_element_type = ir.RankedTensorType(k.type).element_type
336-
# P is the lhs operand of the contraction being replaced, so its
337-
# element type is the precision the rest of the graph expects.
338-
p_element_type = ir.RankedTensorType(
339-
output_op.operands[0].type
340-
).element_type
336+
# P is the lhs of the `@V` contraction, so it carries V's precision --
337+
# both operands have to be narrow for the DPAS.
338+
p_element_type = ir.RankedTensorType(v.type).element_type
341339
out_element_type = ir.RankedTensorType(
342340
output_op.results[0].type
343341
).element_type

‎lighthouse/dialects/transform/transform_ext/ops/sink_normalization_past_contraction.py‎

Lines changed: 38 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -20,12 +20,15 @@ class SinkNormalizationPastContractionOp(
2020
2121
sum_k (A[k] / S) * B[k] == (sum_k A[k] * B[k]) / S
2222
23-
Worth doing when the reduction dim very large compared to the other dims.
23+
Worth doing when the reduction dim is very large compared to the other dims.
24+
25+
The scale is expected inside the contraction's body, i.e. scale is already
26+
fused into the contraction's body.
2427
2528
Before:
2629
```
27-
%p = linalg.generic ins(%e, %l) { arith.divf } (all-parallel)
28-
%o = linalg.generic ins(%p, %v) { mulf, addf } (contraction over k)
30+
%o = linalg.generic ins(%e, %l, %v) { (contraction over k)
31+
divf, mulf, addf }
2932
```
3033
3134
After:
@@ -34,36 +37,29 @@ class SinkNormalizationPastContractionOp(
3437
%n = linalg.generic ins(%o, %l) { arith.divf } (all-parallel)
3538
```
3639
37-
Both payload ops are named explicitly. The op checks that the sink applies and
38-
reports a silenceable error saying why if it does not. It requires:
39-
40-
* Exactly one payload op per handle;
41-
* `contraction` to be a structured linalg op (i.e. a `linalg.generic` or a named
42-
op such as ``linalg.matmul``/``linalg.batch_matmul``) with exactly one
43-
reduction dim, reduction dim is the innermost and, has an identity output map;
44-
* `normalization` to be an all-parallel two-input `linalg.generic` whose body
45-
is a single ``arith.divf``/``arith.mulf`` of two block arguments, with an
46-
identity output map with contraction as its only user;
47-
* The scale, seen from the contraction's iteration space, not to reference the
48-
reduction dim -- the condition that lets it factor out;
49-
* The scale's element type to be convertible to the contraction's accumulator
40+
Op checks that the
41+
sink applies and reports a silenceable error saying why if it does not. It
42+
requires:
43+
44+
* Exactly one payload op for `contraction`;
45+
* `contraction` to be a `linalg.generic` with exactly one reduction dim, that
46+
dim innermost, and an identity output map;
47+
* its body to hold an ``arith.divf``/``arith.mulf`` on two input block
48+
arguments;
49+
* the scale operand not to reference the reduction dim -- the condition that
50+
lets it factor out;
51+
* the scale's element type to be convertible to the contraction's accumulator
5052
type (it is widened when the divide moves).
5153
52-
A named contraction additionally has to be able to read the numerator under the
53-
indexing map already in place, since its own verifier constrains those maps;
54-
generalize it to a `linalg.generic` first if not.
55-
56-
`contraction` is rewritten in place, so a handle to it stays valid.
57-
5854
Args:
59-
normalization: Handle to the scaling linalg.generic.
60-
contraction: Handle to the contraction consuming its result.
55+
contraction: Handle to the contraction carrying the scale.
6156
Returns:
62-
Handle to the new linalg.generic applying the scale after the contraction.
57+
rewritten_contraction: The contraction rebuilt without the scale operand.
58+
sunk_normalization: The new linalg.generic applying the scale after it.
6359
"""
6460

65-
normalization: ext.Operand[transform.AnyOpType]
6661
contraction: ext.Operand[transform.AnyOpType]
62+
rewritten_contraction: ext.Result[transform.AnyOpType[()]] = ext.infer_result()
6763
sunk_normalization: ext.Result[transform.AnyOpType[()]] = ext.infer_result()
6864

6965
@classmethod
@@ -85,23 +81,18 @@ def reject(message: str) -> DiagnosedSilenceableFailure:
8581
)
8682
return DiagnosedSilenceableFailure.SilenceableFailure
8783

88-
payloads = []
89-
for name, handle in (
90-
("normalization", op.normalization),
91-
("contraction", op.contraction),
92-
):
93-
handle_ops = state.get_payload_ops(handle)
94-
if len(handle_ops) != 1:
95-
return reject(
96-
f"expected exactly one payload op for '{name}', got "
97-
f"{len(handle_ops)}"
98-
)
99-
payloads.append(handle_ops[0])
100-
101-
sunk, error = _sink_norm(payloads[0], payloads[1], rewriter)
84+
targets = state.get_payload_ops(op.contraction)
85+
if len(targets) != 1:
86+
return reject(
87+
f"expected exactly one payload op for 'contraction', got "
88+
f"{len(targets)}"
89+
)
90+
91+
result, error = _sink_norm(targets[0], rewriter)
10292
if error is not None:
10393
return reject(error)
104-
results.set_ops(op.sunk_normalization, [sunk.operation])
94+
results.set_ops(op.rewritten_contraction, [result.contraction.operation])
95+
results.set_ops(op.sunk_normalization, [result.normalization.operation])
10596
return DiagnosedSilenceableFailure.Success
10697

10798
@staticmethod
@@ -113,23 +104,18 @@ def allow_repeated_handle_operands(
113104
class MemoryEffectsOpInterfaceModel(ir.MemoryEffectsOpInterface):
114105
@staticmethod
115106
def get_effects(op: ir.Operation):
116-
# The normalization is erased, so its handle is consumed; the
117-
# contraction is rewritten in place, so its handle survives.
118-
operands = list(op.op_operands)
107+
# The in-body form rebuilds the contraction and erases the original, so
108+
# the handle is consumed; `rewritten_contraction` replaces it.
119109
return (
120-
transform.consumes_handle(operands[:1])
121-
+ transform.only_reads_handle(operands[1:])
110+
transform.consumes_handle(op.op_operands)
122111
+ transform.produces_handle(op.results)
123112
+ transform.modifies_payload()
124113
)
125114

126115

127116
def sink_normalization_past_contraction(
128-
normalization: ir.Value[transform.AnyOpType],
129117
contraction: ir.Value[transform.AnyOpType],
130-
) -> ir.Value[transform.AnyOpType]:
118+
) -> tuple[ir.Value[transform.AnyOpType], ir.Value[transform.AnyOpType]]:
131119
"""snake_case wrapper to create SinkNormalizationPastContractionOp."""
132-
op = SinkNormalizationPastContractionOp(
133-
normalization=normalization, contraction=contraction
134-
)
135-
return op.sunk_normalization
120+
op = SinkNormalizationPastContractionOp(contraction=contraction)
121+
return op.rewritten_contraction, op.sunk_normalization

0 commit comments

Comments
 (0)