Skip to content

Commit 3a10839

Browse files
committed
softmax tiling example
1 parent 5500db6 commit 3a10839

1 file changed

Lines changed: 241 additions & 0 deletions

File tree

‎docs/softmax_tiling.md‎

Lines changed: 241 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,241 @@
1+
# Softmax Tiling and Fusion
2+
3+
---
4+
5+
## Stage 0: Initial Softmax
6+
7+
**What we have:** A single high-level `linalg.softmax` operation that computes softmax across dimension 1.
8+
9+
**Softmax formula:** For each row $i$:
10+
11+
$$\text{softmax}(x)_{i,j} = \frac{\exp(x_{i,j} - \max_k x_{i,k})}{\sum_k \exp(x_{i,k} - \max_k x_{i,k})}$$
12+
13+
```mlir
14+
func.func @softmax_v0(%arg0: memref<64x1024xf32>, %arg1: memref<64x1024xf32>) {
15+
%0 = bufferization.to_tensor %arg0 restrict writable : memref<64x1024xf32> to tensor<64x1024xf32>
16+
%1 = tensor.empty() : tensor<64x1024xf32>
17+
%2 = linalg.softmax dimension(1) ins(%0 : tensor<64x1024xf32>) outs(%1 : tensor<64x1024xf32>) -> tensor<64x1024xf32>
18+
bufferization.materialize_in_destination %2 in restrict writable %arg1 : (tensor<64x1024xf32>, memref<64x1024xf32>) -> ()
19+
return
20+
}
21+
```
22+
23+
---
24+
25+
## Stage 1: Softmax Decomposition
26+
27+
**Transform applied:** `transform.structured.decompose_interface`
28+
29+
**What happens:** The high-level softmax operation is decomposed into 5 primitive `linalg.generic` operations:
30+
31+
1. **Max reduction** (`%4`): Find the maximum value along each row
32+
2. **Subtract and exp** (`%5`): Compute `exp(x - max)` elementwise
33+
3. **Fill for sum** (`%6`): Initialize accumulator for sum reduction
34+
4. **Sum reduction** (`%7`): Sum the exp values along each row
35+
5. **Final division** (`%8`): Divide each exp value by the sum
36+
37+
```mlir
38+
func.func @softmax_v0(%arg0: memref<64x1024xf32>, %arg1: memref<64x1024xf32>) {
39+
%0 = bufferization.to_tensor %arg0 restrict writable : memref<64x1024xf32> to tensor<64x1024xf32>
40+
%1 = tensor.empty() : tensor<64x1024xf32>
41+
%2 = tensor.empty() : tensor<64xf32>
42+
%cst = arith.constant 0xFFC00000 : f32
43+
%3 = linalg.fill ins(%cst : f32) outs(%2 : tensor<64xf32>) -> tensor<64xf32>
44+
45+
// 1. Max reduction: compute max along each row
46+
%4 = linalg.generic {
47+
indexing_maps = [affine_map<(d0, d1) -> (d0, d1)>, affine_map<(d0, d1) -> (d0)>],
48+
iterator_types = ["parallel", "reduction"]
49+
} ins(%0 : tensor<64x1024xf32>) outs(%3 : tensor<64xf32>) {
50+
^bb0(%in: f32, %out: f32):
51+
%9 = arith.maxnumf %in, %out : f32
52+
linalg.yield %9 : f32
53+
} -> tensor<64xf32>
54+
55+
// 2. Subtract max and apply exp: exp(x - max)
56+
%5 = linalg.generic {
57+
indexing_maps = [
58+
affine_map<(d0, d1) -> (d0, d1)>,
59+
affine_map<(d0, d1) -> (d0)>,
60+
affine_map<(d0, d1) -> (d0, d1)>
61+
],
62+
iterator_types = ["parallel", "parallel"]
63+
} ins(%0, %4 : tensor<64x1024xf32>, tensor<64xf32>) outs(%1 : tensor<64x1024xf32>) {
64+
^bb0(%in: f32, %in_1: f32, %out: f32):
65+
%9 = arith.subf %in, %in_1 : f32
66+
%10 = math.exp %9 : f32
67+
linalg.yield %10 : f32
68+
} -> tensor<64x1024xf32>
69+
70+
%cst_0 = arith.constant 0.000000e+00 : f32
71+
%6 = linalg.fill ins(%cst_0 : f32) outs(%2 : tensor<64xf32>) -> tensor<64xf32>
72+
73+
// 3. Sum reduction: sum of exp values
74+
%7 = linalg.generic {
75+
indexing_maps = [affine_map<(d0, d1) -> (d0, d1)>, affine_map<(d0, d1) -> (d0)>],
76+
iterator_types = ["parallel", "reduction"]
77+
} ins(%5 : tensor<64x1024xf32>) outs(%6 : tensor<64xf32>) {
78+
^bb0(%in: f32, %out: f32):
79+
%9 = arith.addf %in, %out : f32
80+
linalg.yield %9 : f32
81+
} -> tensor<64xf32>
82+
83+
// 4. Final division: exp(x - max) / sum
84+
%8 = linalg.generic {
85+
indexing_maps = [
86+
affine_map<(d0, d1) -> (d0, d1)>,
87+
affine_map<(d0, d1) -> (d0)>,
88+
affine_map<(d0, d1) -> (d0, d1)>
89+
],
90+
iterator_types = ["parallel", "parallel"]
91+
} ins(%5, %7 : tensor<64x1024xf32>, tensor<64xf32>) outs(%1 : tensor<64x1024xf32>) {
92+
^bb0(%in: f32, %in_1: f32, %out: f32):
93+
%9 = arith.divf %in, %in_1 : f32
94+
linalg.yield %9 : f32
95+
} -> tensor<64x1024xf32>
96+
97+
bufferization.materialize_in_destination %8 in restrict writable %arg1 : (tensor<64x1024xf32>, memref<64x1024xf32>) -> ()
98+
return
99+
}
100+
```
101+
102+
---
103+
104+
## Stage 2: Elementwise Operation Fusion
105+
106+
**Transform applied:** `transform.apply_registered_pass "linalg-fuse-elementwise-ops"` with some modifications.
107+
108+
**What happens:** The compiler fuses operations that can be computed together to reduce memory traffic:
109+
110+
1. The elementwise `exp(x - max)` operation (`%5`) is fused into the sum reduction (`%6`)
111+
2. The elementwise `exp(x - max)` and division are fused into the final output (`%7`)
112+
113+
```mlir
114+
func.func @softmax_v0(%arg0: memref<64x1024xf32>, %arg1: memref<64x1024xf32>) {
115+
%cst = arith.constant 0.000000e+00 : f32
116+
%cst_0 = arith.constant 0xFFC00000 : f32
117+
%0 = bufferization.to_tensor %arg0 restrict writable : memref<64x1024xf32> to tensor<64x1024xf32>
118+
%1 = tensor.empty() : tensor<64x1024xf32>
119+
%2 = tensor.empty() : tensor<64xf32>
120+
%3 = linalg.fill ins(%cst_0 : f32) outs(%2 : tensor<64xf32>) -> tensor<64xf32>
121+
122+
// 1. Max reduction (unchanged)
123+
%4 = linalg.generic {
124+
indexing_maps = [affine_map<(d0, d1) -> (d0, d1)>, affine_map<(d0, d1) -> (d0)>],
125+
iterator_types = ["parallel", "reduction"]
126+
} ins(%0 : tensor<64x1024xf32>) outs(%3 : tensor<64xf32>) {
127+
^bb0(%in: f32, %out: f32):
128+
%8 = arith.maxnumf %in, %out : f32
129+
linalg.yield %8 : f32
130+
} -> tensor<64xf32>
131+
132+
%5 = linalg.fill ins(%cst : f32) outs(%2 : tensor<64xf32>) -> tensor<64xf32>
133+
134+
// 2. Fused: exp(x - max) and sum reduction
135+
// This computes exp on-the-fly during the reduction
136+
%6 = linalg.generic {
137+
indexing_maps = [
138+
affine_map<(d0, d1) -> (d0, d1)>,
139+
affine_map<(d0, d1) -> (d0)>,
140+
affine_map<(d0, d1) -> (d0)>
141+
],
142+
iterator_types = ["parallel", "reduction"]
143+
} ins(%0, %4 : tensor<64x1024xf32>, tensor<64xf32>) outs(%5 : tensor<64xf32>) {
144+
^bb0(%in: f32, %in_1: f32, %out: f32):
145+
%8 = arith.subf %in, %in_1 : f32
146+
%9 = math.exp %8 : f32
147+
%10 = arith.addf %9, %out : f32
148+
linalg.yield %10 : f32
149+
} -> tensor<64xf32>
150+
151+
// 3. Fused: exp(x - max) and division
152+
// This computes exp again and immediately divides by sum
153+
%7 = linalg.generic {
154+
indexing_maps = [
155+
affine_map<(d0, d1) -> (d0, d1)>,
156+
affine_map<(d0, d1) -> (d0)>,
157+
affine_map<(d0, d1) -> (d0)>,
158+
affine_map<(d0, d1) -> (d0, d1)>
159+
],
160+
iterator_types = ["parallel", "parallel"]
161+
} ins(%0, %4, %6 : tensor<64x1024xf32>, tensor<64xf32>, tensor<64xf32>) outs(%1 : tensor<64x1024xf32>) {
162+
^bb0(%in: f32, %in_1: f32, %in_2: f32, %out: f32):
163+
%8 = arith.subf %in, %in_1 : f32
164+
%9 = math.exp %8 : f32
165+
%10 = arith.divf %9, %in_2 : f32
166+
linalg.yield %10 : f32
167+
} -> tensor<64x1024xf32>
168+
169+
bufferization.materialize_in_destination %7 in restrict writable %arg1 : (tensor<64x1024xf32>, memref<64x1024xf32>) -> ()
170+
return
171+
}
172+
```
173+
174+
---
175+
176+
## Stage 3: Max-Sum Reduction Fusion (Online Softmax)
177+
178+
179+
**What happens:** The max reduction and sum reduction are fused into a single pass over the data using the "online softmax" algorithm. This algorithm maintains running max and running sum values, applying a correction factor when the max changes.
180+
181+
**Algorithm:**
182+
```
183+
For each element x in the row:
184+
new_max = max(current_max, x)
185+
correction = exp(old_max - new_max)
186+
corrected_sum = current_sum * correction
187+
new_sum = corrected_sum + exp(x - new_max)
188+
```
189+
190+
```mlir
191+
#map = affine_map<(d0, d1) -> (d0, d1)>
192+
#map1 = affine_map<(d0, d1) -> (d0)>
193+
194+
func.func @softmax_v2(%arg0: memref<64x1024xf32>, %arg1: memref<64x1024xf32>) {
195+
%cst = arith.constant 0.000000e+00 : f32
196+
%cst_0 = arith.constant 0xFFC00000 : f32
197+
%0 = bufferization.to_tensor %arg0 restrict writable : memref<64x1024xf32> to tensor<64x1024xf32>
198+
%1 = tensor.empty() : tensor<64x1024xf32>
199+
%2 = tensor.empty() : tensor<64xf32>
200+
%3 = linalg.fill ins(%cst_0 : f32) outs(%2 : tensor<64xf32>) -> tensor<64xf32>
201+
%4 = linalg.fill ins(%cst : f32) outs(%2 : tensor<64xf32>) -> tensor<64xf32>
202+
203+
// Fused max and sum reduction with online correction
204+
%5:2 = linalg.generic {
205+
indexing_maps = [#map, #map1, #map1],
206+
iterator_types = ["parallel", "reduction"]
207+
} ins(%0 : tensor<64x1024xf32>) outs(%3, %4 : tensor<64xf32>, tensor<64xf32>) {
208+
^bb0(%in: f32, %out_max: f32, %out_sum: f32):
209+
// Update max
210+
%new_max = arith.maxnumf %in, %out_max : f32
211+
212+
// Compute correction factor for sum when max changes
213+
// correction = exp(old_max - new_max)
214+
%max_diff = arith.subf %out_max, %new_max : f32
215+
%correction = math.exp %max_diff : f32
216+
%corrected_sum = arith.mulf %out_sum, %correction : f32
217+
218+
// Add new contribution: exp(value - new_max)
219+
%val_diff = arith.subf %in, %new_max : f32
220+
%exp_val = math.exp %val_diff : f32
221+
%new_sum = arith.addf %corrected_sum, %exp_val : f32
222+
223+
linalg.yield %new_max, %new_sum : f32, f32
224+
} -> (tensor<64xf32>, tensor<64xf32>)
225+
226+
// Final division to compute softmax
227+
%6 = linalg.generic {
228+
indexing_maps = [#map, #map1, #map1, #map],
229+
iterator_types = ["parallel", "parallel"]
230+
} ins(%0, %5#0, %5#1 : tensor<64x1024xf32>, tensor<64xf32>, tensor<64xf32>) outs(%1 : tensor<64x1024xf32>) {
231+
^bb0(%in: f32, %in_max: f32, %in_sum: f32, %out: f32):
232+
%7 = arith.subf %in, %in_max : f32
233+
%8 = math.exp %7 : f32
234+
%9 = arith.divf %8, %in_sum : f32
235+
linalg.yield %9 : f32
236+
} -> tensor<64x1024xf32>
237+
238+
bufferization.materialize_in_destination %6 in restrict writable %arg1 : (tensor<64x1024xf32>, memref<64x1024xf32>) -> ()
239+
return
240+
}
241+
```

0 commit comments

Comments
 (0)