Skip to content

Commit 909d77e

Browse files
committed
2D fused loop is possible version. needs e2e testing..
1 parent 0cd22b6 commit 909d77e

2 files changed

Lines changed: 316 additions & 167 deletions

File tree

‎lighthouse/ingress/mlir_gen/gpu_attention_payload.py‎

Lines changed: 92 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,15 @@
44

55
from mlir import ir
66
from mlir.dialects import arith, bufferization, linalg, memref, tensor
7+
from mlir.dialects import math as math_dialect
78

89
from lighthouse.utils.mlir import func_cif
9-
from lighthouse.ingress.mlir_gen.utils import emit_buf_to_tensor
10+
from lighthouse.ingress.mlir_gen.utils import (
11+
affine_map,
12+
emit_buf_to_tensor,
13+
parallel,
14+
reduction,
15+
)
1016

1117

1218
def generate_gpu_attention_payload(
@@ -23,6 +29,25 @@ def generate_gpu_attention_payload(
2329
Computes attention:
2430
output = softmax(Q @ K^T / sqrt(n_head)) @ V
2531
32+
The softmax is written out explicitly and, unlike `linalg.softmax`, with the
33+
normalizing divide deferred *past* the attention_weights @ V contraction:
34+
35+
S[b, i, j] = (Q @ K^T)[b, i, j] * scale
36+
m[b, i] = max_j S[b, i, j]
37+
p[b, i, j] = exp(S[b, i, j] - m[b, i])
38+
l[b, i] = sum_j p[b, i, j]
39+
O[b, i, d] = sum_j p[b, i, j] * V[b, j, d]
40+
out = O[b, i, d] / l[b, i]
41+
42+
which is the same value as `softmax(S) @ V` because `l` does not depend on
43+
the contracted axis `j`. This is the form fused attention needs: `p` feeds
44+
the contraction directly, so both `l` and `O` are reductions of `p` over `j`
45+
and can be turned into running accumulators of a single loop over `j`.
46+
47+
`p` is a single op shared by both `l` and `O`; the reduction fusion clones it
48+
so that each of those two reductions has its own elementwise term to move
49+
into the loop.
50+
2651
Args:
2752
func_name: Name of the payload function
2853
Z: Batch size
@@ -105,24 +130,78 @@ def payload(output, Q_arg, K_arg, V_arg):
105130
scaled_qkt_init = tensor.empty(qkt_shape_3d, dtype)
106131
scaled_qkt = linalg.mul(qkt, scale_tensor, outs=[scaled_qkt_init])
107132

108-
# Step 4: Apply softmax along the last dimension (dim=2 in 3D)
109-
softmax_init = tensor.empty(qkt_shape_3d, dtype)
110-
attention_weights = linalg.softmax(
111-
result=[ir.RankedTensorType.get(qkt_shape_3d, dtype)],
112-
input=scaled_qkt,
113-
output=softmax_init,
114-
dimension=2,
133+
# Step 4: Softmax numerator, written out over the iteration space
134+
# (b, i, j) with j -- the key/value axis -- reduced.
135+
all_parallel_map = affine_map(
136+
3,
137+
[
138+
ir.AffineDimExpr.get(0),
139+
ir.AffineDimExpr.get(1),
140+
ir.AffineDimExpr.get(2),
141+
],
142+
)
143+
row_map = affine_map(3, [ir.AffineDimExpr.get(0), ir.AffineDimExpr.get(1)])
144+
row_shape_2d = (batch_dim, n_ctx)
145+
146+
# 4a) Running max: m[b, i] = max_j S[b, i, j]
147+
neg_inf = arith.constant(dtype, float("-inf"))
148+
max_init = linalg.fill(neg_inf, outs=[tensor.empty(row_shape_2d, dtype)])
149+
150+
@linalg.generic(
151+
[scaled_qkt],
152+
[max_init],
153+
[all_parallel_map, row_map],
154+
[parallel, parallel, reduction],
155+
)
156+
def row_max(s, acc):
157+
return arith.MaximumFOp(s, acc)
158+
159+
# 4b) Shifted exponential: p[b, i, j] = exp(S[b, i, j] - m[b, i])
160+
#
161+
# A single op, shared by both consumers below.
162+
# `transform.structured.fuse_dependant_reduction_ops` needs each
163+
# consumer reduction to have its own elementwise term to move into the
164+
# reduction loop, and clones this one as it fuses them.
165+
@linalg.generic(
166+
[scaled_qkt, row_max],
167+
[tensor.empty(qkt_shape_3d, dtype)],
168+
[all_parallel_map, row_map, all_parallel_map],
169+
[parallel, parallel, parallel],
115170
)
171+
def attention_weights(s, m, _out):
172+
return math_dialect.ExpOp(arith.SubFOp(s, m).result)
116173

117-
# Step 5: Multiply attention weights by V using batch_matmul
118-
# attention_weights: (batch_dim, n_ctx, n_ctx) @ V: (batch_dim, n_ctx, n_head)
174+
# 4c) Normalizer: l[b, i] = sum_j p[b, i, j]
175+
sum_init = linalg.fill(zero, outs=[tensor.empty(row_shape_2d, dtype)])
176+
177+
@linalg.generic(
178+
[attention_weights],
179+
[sum_init],
180+
[all_parallel_map, row_map],
181+
[parallel, parallel, reduction],
182+
)
183+
def row_sum(p, acc):
184+
return arith.AddFOp(p, acc)
185+
186+
# Step 5: Contract the (unnormalized) weights with V using batch_matmul
187+
# p: (batch_dim, n_ctx, n_ctx) @ V: (batch_dim, n_ctx, n_head)
119188
# Result: (batch_dim, n_ctx, n_head)
189+
pv_init = tensor.empty(collapsed_shape_3d, dtype)
190+
pv_init_filled = linalg.fill(zero, outs=[pv_init])
191+
192+
pv = linalg.batch_matmul(attention_weights, V_3d, outs=[pv_init_filled])
193+
194+
# Step 6: Deferred normalization: out[b, i, d] = O[b, i, d] / l[b, i]
120195
output_3d_init = tensor.empty(collapsed_shape_3d, dtype)
121-
output_3d_init_filled = linalg.fill(zero, outs=[output_3d_init])
122196

123-
result_3d = linalg.batch_matmul(
124-
attention_weights, V_3d, outs=[output_3d_init_filled]
197+
@linalg.generic(
198+
[pv, row_sum],
199+
[output_3d_init],
200+
[all_parallel_map, row_map, all_parallel_map],
201+
[parallel, parallel, parallel],
125202
)
203+
def result_3d(o, norm, _out):
204+
return arith.DivFOp(o, norm)
126205

127206
# Materialize 3D result back to 3D output memref
128207
bufferization.materialize_in_destination(

0 commit comments

Comments
 (0)