Skip to content

Commit 226a6b7

Browse files
committed
modify nanoGPT to use new form
1 parent 30bc10e commit 226a6b7

4 files changed

Lines changed: 255 additions & 595 deletions

File tree

‎examples/xegpu/nanoGPT_payload.py‎

Lines changed: 75 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -21,12 +21,13 @@
2121
"""
2222

2323
from mlir import ir
24-
from mlir.dialects import linalg, bufferization, tensor, arith, gpu, memref
24+
from mlir.dialects import linalg, bufferization, tensor, arith, gpu, math, memref
2525

2626
from lighthouse.ingress.mlir_gen.utils import (
2727
emit_buf_to_tensor,
2828
affine_map,
2929
parallel,
30+
reduction,
3031
)
3132
from lighthouse.ingress.mlir_gen.gpu_utils import emit_gpu_util_funcs
3233
from lighthouse.ingress.mlir_gen.gpu_layer_norm_payload import emit_layer_norm_generics
@@ -229,11 +230,24 @@ def heads_view(self, buf2d, n_ctx, n_head, d_head):
229230
def attention_4d(
230231
self, Qh, Kh, Vh, n_head, n_ctx, d_head, out_view, out_view_memref
231232
):
232-
# Emits the SAME linalg op sequence as generate_gpu_attention_payload
233-
# (batch_matmul QK^T -> scale-mul -> softmax -> batch_matmul @V), so the
234-
# fused-attention schedule's matchers/rewrite apply verbatim. After the
235-
# per-region fused tiling, all these ops fuse into one scf.forall -> one
236-
# GPU kernel (the flash/online-softmax kernel). Counts as one 'fa'.
233+
# batch_matmul QK^T -> scale-mul -> softmax -> batch_matmul @V, with the
234+
# softmax spelled out in the flash-attention form:
235+
#
236+
# m = max_j s l = sum_j p
237+
# p = exp(s - m) o = p @ V out = o / l
238+
#
239+
# i.e. the normalizing divide comes *after* the contraction. That is
240+
# algebraically identical to `softmax(s) @ V` -- dividing by the per-row `l`
241+
# commutes with a contraction that reduces the other axis -- but it leaves
242+
# the `max -> exp -> {sum, @V}` dependency chain explicit, which is what
243+
# `transform_ext.fuse_dependant_reduction_ops` consumes to derive the online
244+
# one-pass loop (see `_fuse_attention_in_region` in nanoGPT_schedule.py).
245+
# A `linalg.softmax` would not do: its decomposition normalizes *before* the
246+
# contraction, leaving @V reading the normalized P and breaking the chain.
247+
# Matches the sequence `generate_gpu_attention_payload` emits.
248+
#
249+
# After the per-region fused tiling, all these ops fuse into one scf.forall
250+
# -> one GPU kernel (the flash/online-softmax kernel). Counts as one 'fa'.
237251
# Inputs Qh/Kh/Vh are (n_head,n_ctx,d_head) f16 strided views (heads_view); the @V result
238252
# is materialized into `out_view`, a (n_head,n_ctx,d_head) strided view of a (n_ctx,n_embd) buffer,
239253
# so the merge back to 2D is also a free view (no from_heads kernel).
@@ -252,15 +266,62 @@ def attention_4d(
252266
scaled = linalg.mul(
253267
qkt, scale_t, outs=[tensor.empty((n_head, n_ctx, n_ctx), f16)]
254268
)
255-
aw = linalg.softmax(
256-
result=[ir.RankedTensorType.get((n_head, n_ctx, n_ctx), f16)],
257-
input=scaled,
258-
output=tensor.empty((n_head, n_ctx, n_ctx), f16),
259-
dimension=2,
269+
270+
# (head, row, col) -> (head, row, col) and -> (head, row): the per-row
271+
# statistics are broadcast over the reduced axis.
272+
dims = [ir.AffineDimExpr.get(i) for i in range(3)]
273+
ew_map = affine_map(3, dims)
274+
row_map = affine_map(3, dims[:2])
275+
row_shape = (n_head, n_ctx)
276+
277+
# m = max_j s
278+
neg_inf = arith.constant(f16, float("-inf"))
279+
max_acc = linalg.fill(neg_inf, outs=[tensor.empty(row_shape, f16)])
280+
281+
@linalg.generic(
282+
[scaled], [max_acc], [ew_map, row_map], [parallel, parallel, reduction]
260283
)
261-
# @V: (n_head,n_ctx,n_ctx) @ (n_head,n_ctx,d_head) -> (n_head,n_ctx,d_head) f16, materialized into the (n_ctx,n_embd) view.
262-
out_filled = linalg.fill(zero, outs=[out_view])
263-
out = linalg.batch_matmul(aw, Vh, outs=[out_filled])
284+
def row_max(s, acc):
285+
return arith.MaximumFOp(s, acc)
286+
287+
# p = exp(s - m), read by both the row sum and the @V contraction.
288+
@linalg.generic(
289+
[scaled, row_max],
290+
[tensor.empty((n_head, n_ctx, n_ctx), f16)],
291+
[ew_map, row_map, ew_map],
292+
[parallel, parallel, parallel],
293+
)
294+
def probs(s, m, out):
295+
return math.ExpOp(arith.SubFOp(s, m).result)
296+
297+
# l = sum_j p
298+
sum_acc = linalg.fill(zero, outs=[tensor.empty(row_shape, f16)])
299+
300+
@linalg.generic(
301+
[probs], [sum_acc], [ew_map, row_map], [parallel, parallel, reduction]
302+
)
303+
def row_sum(p, acc):
304+
return arith.AddFOp(p, acc)
305+
306+
# @V: (n_head,n_ctx,n_ctx) @ (n_head,n_ctx,d_head) -> (n_head,n_ctx,d_head)
307+
# f16, still unnormalized.
308+
unnorm_init = linalg.fill(
309+
zero, outs=[tensor.empty((n_head, n_ctx, d_head), f16)]
310+
)
311+
unnormalized = linalg.batch_matmul(probs, Vh, outs=[unnorm_init])
312+
313+
# out = o / l, the deferred normalization, materialized into the
314+
# (n_ctx,n_embd) view. `l` broadcasts over d_head, the contraction's free
315+
# axis.
316+
@linalg.generic(
317+
[unnormalized, row_sum],
318+
[out_view],
319+
[ew_map, row_map, ew_map],
320+
[parallel, parallel, parallel],
321+
)
322+
def out(o, denom, dst):
323+
return arith.DivFOp(o, denom)
324+
264325
bufferization.materialize_in_destination(
265326
None, out, out_view_memref, restrict=True, writable=True
266327
)

0 commit comments

Comments
 (0)