44
55from mlir import ir
66from mlir .dialects import arith , bufferization , linalg , memref , tensor
7+ from mlir .dialects import math as math_dialect
78
89from lighthouse .utils .mlir import func_cif
9- from lighthouse .ingress .mlir_gen .utils import emit_buf_to_tensor
10+ from lighthouse .ingress .mlir_gen .utils import (
11+ affine_map ,
12+ emit_buf_to_tensor ,
13+ parallel ,
14+ reduction ,
15+ )
1016
1117
1218def generate_gpu_attention_payload (
@@ -23,6 +29,25 @@ def generate_gpu_attention_payload(
2329 Computes attention:
2430 output = softmax(Q @ K^T / sqrt(n_head)) @ V
2531
32+ The softmax is written out explicitly and, unlike `linalg.softmax`, with the
33+ normalizing divide deferred *past* the attention_weights @ V contraction:
34+
35+ S[b, i, j] = (Q @ K^T)[b, i, j] * scale
36+ m[b, i] = max_j S[b, i, j]
37+ p[b, i, j] = exp(S[b, i, j] - m[b, i])
38+ l[b, i] = sum_j p[b, i, j]
39+ O[b, i, d] = sum_j p[b, i, j] * V[b, j, d]
40+ out = O[b, i, d] / l[b, i]
41+
42+ which is the same value as `softmax(S) @ V` because `l` does not depend on
43+ the contracted axis `j`. This is the form fused attention needs: `p` feeds
44+ the contraction directly, so both `l` and `O` are reductions of `p` over `j`
45+ and can be turned into running accumulators of a single loop over `j`.
46+
47+ `p` is a single op shared by both `l` and `O`; the reduction fusion clones it
48+ so that each of those two reductions has its own elementwise term to move
49+ into the loop.
50+
2651 Args:
2752 func_name: Name of the payload function
2853 Z: Batch size
@@ -105,24 +130,78 @@ def payload(output, Q_arg, K_arg, V_arg):
105130 scaled_qkt_init = tensor .empty (qkt_shape_3d , dtype )
106131 scaled_qkt = linalg .mul (qkt , scale_tensor , outs = [scaled_qkt_init ])
107132
108- # Step 4: Apply softmax along the last dimension (dim=2 in 3D)
109- softmax_init = tensor .empty (qkt_shape_3d , dtype )
110- attention_weights = linalg .softmax (
111- result = [ir .RankedTensorType .get (qkt_shape_3d , dtype )],
112- input = scaled_qkt ,
113- output = softmax_init ,
114- dimension = 2 ,
133+ # Step 4: Softmax numerator, written out over the iteration space
134+ # (b, i, j) with j -- the key/value axis -- reduced.
135+ all_parallel_map = affine_map (
136+ 3 ,
137+ [
138+ ir .AffineDimExpr .get (0 ),
139+ ir .AffineDimExpr .get (1 ),
140+ ir .AffineDimExpr .get (2 ),
141+ ],
142+ )
143+ row_map = affine_map (3 , [ir .AffineDimExpr .get (0 ), ir .AffineDimExpr .get (1 )])
144+ row_shape_2d = (batch_dim , n_ctx )
145+
146+ # 4a) Running max: m[b, i] = max_j S[b, i, j]
147+ neg_inf = arith .constant (dtype , float ("-inf" ))
148+ max_init = linalg .fill (neg_inf , outs = [tensor .empty (row_shape_2d , dtype )])
149+
150+ @linalg .generic (
151+ [scaled_qkt ],
152+ [max_init ],
153+ [all_parallel_map , row_map ],
154+ [parallel , parallel , reduction ],
155+ )
156+ def row_max (s , acc ):
157+ return arith .MaximumFOp (s , acc )
158+
159+ # 4b) Shifted exponential: p[b, i, j] = exp(S[b, i, j] - m[b, i])
160+ #
161+ # A single op, shared by both consumers below.
162+ # `transform.structured.fuse_dependant_reduction_ops` needs each
163+ # consumer reduction to have its own elementwise term to move into the
164+ # reduction loop, and clones this one as it fuses them.
165+ @linalg .generic (
166+ [scaled_qkt , row_max ],
167+ [tensor .empty (qkt_shape_3d , dtype )],
168+ [all_parallel_map , row_map , all_parallel_map ],
169+ [parallel , parallel , parallel ],
115170 )
171+ def attention_weights (s , m , _out ):
172+ return math_dialect .ExpOp (arith .SubFOp (s , m ).result )
116173
117- # Step 5: Multiply attention weights by V using batch_matmul
118- # attention_weights: (batch_dim, n_ctx, n_ctx) @ V: (batch_dim, n_ctx, n_head)
174+ # 4c) Normalizer: l[b, i] = sum_j p[b, i, j]
175+ sum_init = linalg .fill (zero , outs = [tensor .empty (row_shape_2d , dtype )])
176+
177+ @linalg .generic (
178+ [attention_weights ],
179+ [sum_init ],
180+ [all_parallel_map , row_map ],
181+ [parallel , parallel , reduction ],
182+ )
183+ def row_sum (p , acc ):
184+ return arith .AddFOp (p , acc )
185+
186+ # Step 5: Contract the (unnormalized) weights with V using batch_matmul
187+ # p: (batch_dim, n_ctx, n_ctx) @ V: (batch_dim, n_ctx, n_head)
119188 # Result: (batch_dim, n_ctx, n_head)
189+ pv_init = tensor .empty (collapsed_shape_3d , dtype )
190+ pv_init_filled = linalg .fill (zero , outs = [pv_init ])
191+
192+ pv = linalg .batch_matmul (attention_weights , V_3d , outs = [pv_init_filled ])
193+
194+ # Step 6: Deferred normalization: out[b, i, d] = O[b, i, d] / l[b, i]
120195 output_3d_init = tensor .empty (collapsed_shape_3d , dtype )
121- output_3d_init_filled = linalg .fill (zero , outs = [output_3d_init ])
122196
123- result_3d = linalg .batch_matmul (
124- attention_weights , V_3d , outs = [output_3d_init_filled ]
197+ @linalg .generic (
198+ [pv , row_sum ],
199+ [output_3d_init ],
200+ [all_parallel_map , row_map , all_parallel_map ],
201+ [parallel , parallel , parallel ],
125202 )
203+ def result_3d (o , norm , _out ):
204+ return arith .DivFOp (o , norm )
126205
127206 # Materialize 3D result back to 3D output memref
128207 bufferization .materialize_in_destination (
0 commit comments