2121"""
2222
2323from mlir import ir
24- from mlir .dialects import linalg , bufferization , tensor , arith , gpu , memref
24+ from mlir .dialects import linalg , bufferization , tensor , arith , gpu , math , memref
2525
2626from lighthouse .ingress .mlir_gen .utils import (
2727 emit_buf_to_tensor ,
2828 affine_map ,
2929 parallel ,
30+ reduction ,
3031)
3132from lighthouse .ingress .mlir_gen .gpu_utils import emit_gpu_util_funcs
3233from lighthouse .ingress .mlir_gen .gpu_layer_norm_payload import emit_layer_norm_generics
@@ -229,11 +230,24 @@ def heads_view(self, buf2d, n_ctx, n_head, d_head):
229230 def attention_4d (
230231 self , Qh , Kh , Vh , n_head , n_ctx , d_head , out_view , out_view_memref
231232 ):
232- # Emits the SAME linalg op sequence as generate_gpu_attention_payload
233- # (batch_matmul QK^T -> scale-mul -> softmax -> batch_matmul @V), so the
234- # fused-attention schedule's matchers/rewrite apply verbatim. After the
235- # per-region fused tiling, all these ops fuse into one scf.forall -> one
236- # GPU kernel (the flash/online-softmax kernel). Counts as one 'fa'.
233+ # batch_matmul QK^T -> scale-mul -> softmax -> batch_matmul @V, with the
234+ # softmax spelled out in the flash-attention form:
235+ #
236+ # m = max_j s l = sum_j p
237+ # p = exp(s - m) o = p @ V out = o / l
238+ #
239+ # i.e. the normalizing divide comes *after* the contraction. That is
240+ # algebraically identical to `softmax(s) @ V` -- dividing by the per-row `l`
241+ # commutes with a contraction that reduces the other axis -- but it leaves
242+ # the `max -> exp -> {sum, @V}` dependency chain explicit, which is what
243+ # `transform_ext.fuse_dependant_reduction_ops` consumes to derive the online
244+ # one-pass loop (see `_fuse_attention_in_region` in nanoGPT_schedule.py).
245+ # A `linalg.softmax` would not do: its decomposition normalizes *before* the
246+ # contraction, leaving @V reading the normalized P and breaking the chain.
247+ # Matches the sequence `generate_gpu_attention_payload` emits.
248+ #
249+ # After the per-region fused tiling, all these ops fuse into one scf.forall
250+ # -> one GPU kernel (the flash/online-softmax kernel). Counts as one 'fa'.
237251 # Inputs Qh/Kh/Vh are (n_head,n_ctx,d_head) f16 strided views (heads_view); the @V result
238252 # is materialized into `out_view`, a (n_head,n_ctx,d_head) strided view of a (n_ctx,n_embd) buffer,
239253 # so the merge back to 2D is also a free view (no from_heads kernel).
@@ -252,15 +266,62 @@ def attention_4d(
252266 scaled = linalg .mul (
253267 qkt , scale_t , outs = [tensor .empty ((n_head , n_ctx , n_ctx ), f16 )]
254268 )
255- aw = linalg .softmax (
256- result = [ir .RankedTensorType .get ((n_head , n_ctx , n_ctx ), f16 )],
257- input = scaled ,
258- output = tensor .empty ((n_head , n_ctx , n_ctx ), f16 ),
259- dimension = 2 ,
269+
270+ # (head, row, col) -> (head, row, col) and -> (head, row): the per-row
271+ # statistics are broadcast over the reduced axis.
272+ dims = [ir .AffineDimExpr .get (i ) for i in range (3 )]
273+ ew_map = affine_map (3 , dims )
274+ row_map = affine_map (3 , dims [:2 ])
275+ row_shape = (n_head , n_ctx )
276+
277+ # m = max_j s
278+ neg_inf = arith .constant (f16 , float ("-inf" ))
279+ max_acc = linalg .fill (neg_inf , outs = [tensor .empty (row_shape , f16 )])
280+
281+ @linalg .generic (
282+ [scaled ], [max_acc ], [ew_map , row_map ], [parallel , parallel , reduction ]
260283 )
261- # @V: (n_head,n_ctx,n_ctx) @ (n_head,n_ctx,d_head) -> (n_head,n_ctx,d_head) f16, materialized into the (n_ctx,n_embd) view.
262- out_filled = linalg .fill (zero , outs = [out_view ])
263- out = linalg .batch_matmul (aw , Vh , outs = [out_filled ])
284+ def row_max (s , acc ):
285+ return arith .MaximumFOp (s , acc )
286+
287+ # p = exp(s - m), read by both the row sum and the @V contraction.
288+ @linalg .generic (
289+ [scaled , row_max ],
290+ [tensor .empty ((n_head , n_ctx , n_ctx ), f16 )],
291+ [ew_map , row_map , ew_map ],
292+ [parallel , parallel , parallel ],
293+ )
294+ def probs (s , m , out ):
295+ return math .ExpOp (arith .SubFOp (s , m ).result )
296+
297+ # l = sum_j p
298+ sum_acc = linalg .fill (zero , outs = [tensor .empty (row_shape , f16 )])
299+
300+ @linalg .generic (
301+ [probs ], [sum_acc ], [ew_map , row_map ], [parallel , parallel , reduction ]
302+ )
303+ def row_sum (p , acc ):
304+ return arith .AddFOp (p , acc )
305+
306+ # @V: (n_head,n_ctx,n_ctx) @ (n_head,n_ctx,d_head) -> (n_head,n_ctx,d_head)
307+ # f16, still unnormalized.
308+ unnorm_init = linalg .fill (
309+ zero , outs = [tensor .empty ((n_head , n_ctx , d_head ), f16 )]
310+ )
311+ unnormalized = linalg .batch_matmul (probs , Vh , outs = [unnorm_init ])
312+
313+ # out = o / l, the deferred normalization, materialized into the
314+ # (n_ctx,n_embd) view. `l` broadcasts over d_head, the contraction's free
315+ # axis.
316+ @linalg .generic (
317+ [unnormalized , row_sum ],
318+ [out_view ],
319+ [ew_map , row_map , ew_map ],
320+ [parallel , parallel , parallel ],
321+ )
322+ def out (o , denom , dst ):
323+ return arith .DivFOp (o , denom )
324+
264325 bufferization .materialize_in_destination (
265326 None , out , out_view_memref , restrict = True , writable = True
266327 )
0 commit comments