|
| 1 | +# Softmax Tiling and Fusion |
| 2 | + |
| 3 | +--- |
| 4 | + |
| 5 | +## Stage 0: Initial Softmax |
| 6 | + |
| 7 | +**What we have:** A single high-level `linalg.softmax` operation that computes softmax across dimension 1. |
| 8 | + |
| 9 | +**Softmax formula:** For each row $i$: |
| 10 | + |
| 11 | +$$\text{softmax}(x)_{i,j} = \frac{\exp(x_{i,j} - \max_k x_{i,k})}{\sum_k \exp(x_{i,k} - \max_k x_{i,k})}$$ |
| 12 | + |
| 13 | +```mlir |
| 14 | +func.func @softmax_v0(%arg0: memref<64x1024xf32>, %arg1: memref<64x1024xf32>) { |
| 15 | + %0 = bufferization.to_tensor %arg0 restrict writable : memref<64x1024xf32> to tensor<64x1024xf32> |
| 16 | + %1 = tensor.empty() : tensor<64x1024xf32> |
| 17 | + %2 = linalg.softmax dimension(1) ins(%0 : tensor<64x1024xf32>) outs(%1 : tensor<64x1024xf32>) -> tensor<64x1024xf32> |
| 18 | + bufferization.materialize_in_destination %2 in restrict writable %arg1 : (tensor<64x1024xf32>, memref<64x1024xf32>) -> () |
| 19 | + return |
| 20 | +} |
| 21 | +``` |
| 22 | + |
| 23 | +--- |
| 24 | + |
| 25 | +## Stage 1: Softmax Decomposition |
| 26 | + |
| 27 | +**Transform applied:** `transform.structured.decompose_interface` |
| 28 | + |
| 29 | +**What happens:** The high-level softmax operation is decomposed into 5 primitive `linalg.generic` operations: |
| 30 | + |
| 31 | +1. **Max reduction** (`%4`): Find the maximum value along each row |
| 32 | +2. **Subtract and exp** (`%5`): Compute `exp(x - max)` elementwise |
| 33 | +3. **Fill for sum** (`%6`): Initialize accumulator for sum reduction |
| 34 | +4. **Sum reduction** (`%7`): Sum the exp values along each row |
| 35 | +5. **Final division** (`%8`): Divide each exp value by the sum |
| 36 | + |
| 37 | +```mlir |
| 38 | +func.func @softmax_v0(%arg0: memref<64x1024xf32>, %arg1: memref<64x1024xf32>) { |
| 39 | + %0 = bufferization.to_tensor %arg0 restrict writable : memref<64x1024xf32> to tensor<64x1024xf32> |
| 40 | + %1 = tensor.empty() : tensor<64x1024xf32> |
| 41 | + %2 = tensor.empty() : tensor<64xf32> |
| 42 | + %cst = arith.constant 0xFFC00000 : f32 |
| 43 | + %3 = linalg.fill ins(%cst : f32) outs(%2 : tensor<64xf32>) -> tensor<64xf32> |
| 44 | +
|
| 45 | + // 1. Max reduction: compute max along each row |
| 46 | + %4 = linalg.generic { |
| 47 | + indexing_maps = [affine_map<(d0, d1) -> (d0, d1)>, affine_map<(d0, d1) -> (d0)>], |
| 48 | + iterator_types = ["parallel", "reduction"] |
| 49 | + } ins(%0 : tensor<64x1024xf32>) outs(%3 : tensor<64xf32>) { |
| 50 | + ^bb0(%in: f32, %out: f32): |
| 51 | + %9 = arith.maxnumf %in, %out : f32 |
| 52 | + linalg.yield %9 : f32 |
| 53 | + } -> tensor<64xf32> |
| 54 | +
|
| 55 | + // 2. Subtract max and apply exp: exp(x - max) |
| 56 | + %5 = linalg.generic { |
| 57 | + indexing_maps = [ |
| 58 | + affine_map<(d0, d1) -> (d0, d1)>, |
| 59 | + affine_map<(d0, d1) -> (d0)>, |
| 60 | + affine_map<(d0, d1) -> (d0, d1)> |
| 61 | + ], |
| 62 | + iterator_types = ["parallel", "parallel"] |
| 63 | + } ins(%0, %4 : tensor<64x1024xf32>, tensor<64xf32>) outs(%1 : tensor<64x1024xf32>) { |
| 64 | + ^bb0(%in: f32, %in_1: f32, %out: f32): |
| 65 | + %9 = arith.subf %in, %in_1 : f32 |
| 66 | + %10 = math.exp %9 : f32 |
| 67 | + linalg.yield %10 : f32 |
| 68 | + } -> tensor<64x1024xf32> |
| 69 | +
|
| 70 | + %cst_0 = arith.constant 0.000000e+00 : f32 |
| 71 | + %6 = linalg.fill ins(%cst_0 : f32) outs(%2 : tensor<64xf32>) -> tensor<64xf32> |
| 72 | +
|
| 73 | + // 3. Sum reduction: sum of exp values |
| 74 | + %7 = linalg.generic { |
| 75 | + indexing_maps = [affine_map<(d0, d1) -> (d0, d1)>, affine_map<(d0, d1) -> (d0)>], |
| 76 | + iterator_types = ["parallel", "reduction"] |
| 77 | + } ins(%5 : tensor<64x1024xf32>) outs(%6 : tensor<64xf32>) { |
| 78 | + ^bb0(%in: f32, %out: f32): |
| 79 | + %9 = arith.addf %in, %out : f32 |
| 80 | + linalg.yield %9 : f32 |
| 81 | + } -> tensor<64xf32> |
| 82 | +
|
| 83 | + // 4. Final division: exp(x - max) / sum |
| 84 | + %8 = linalg.generic { |
| 85 | + indexing_maps = [ |
| 86 | + affine_map<(d0, d1) -> (d0, d1)>, |
| 87 | + affine_map<(d0, d1) -> (d0)>, |
| 88 | + affine_map<(d0, d1) -> (d0, d1)> |
| 89 | + ], |
| 90 | + iterator_types = ["parallel", "parallel"] |
| 91 | + } ins(%5, %7 : tensor<64x1024xf32>, tensor<64xf32>) outs(%1 : tensor<64x1024xf32>) { |
| 92 | + ^bb0(%in: f32, %in_1: f32, %out: f32): |
| 93 | + %9 = arith.divf %in, %in_1 : f32 |
| 94 | + linalg.yield %9 : f32 |
| 95 | + } -> tensor<64x1024xf32> |
| 96 | +
|
| 97 | + bufferization.materialize_in_destination %8 in restrict writable %arg1 : (tensor<64x1024xf32>, memref<64x1024xf32>) -> () |
| 98 | + return |
| 99 | +} |
| 100 | +``` |
| 101 | + |
| 102 | +--- |
| 103 | + |
| 104 | +## Stage 2: Elementwise Operation Fusion |
| 105 | + |
| 106 | +**Transform applied:** `transform.apply_registered_pass "linalg-fuse-elementwise-ops"` with some modifications. |
| 107 | + |
| 108 | +**What happens:** The compiler fuses operations that can be computed together to reduce memory traffic: |
| 109 | + |
| 110 | +1. The elementwise `exp(x - max)` operation (`%5`) is fused into the sum reduction (`%6`) |
| 111 | +2. The elementwise `exp(x - max)` and division are fused into the final output (`%7`) |
| 112 | + |
| 113 | +```mlir |
| 114 | +func.func @softmax_v0(%arg0: memref<64x1024xf32>, %arg1: memref<64x1024xf32>) { |
| 115 | + %cst = arith.constant 0.000000e+00 : f32 |
| 116 | + %cst_0 = arith.constant 0xFFC00000 : f32 |
| 117 | + %0 = bufferization.to_tensor %arg0 restrict writable : memref<64x1024xf32> to tensor<64x1024xf32> |
| 118 | + %1 = tensor.empty() : tensor<64x1024xf32> |
| 119 | + %2 = tensor.empty() : tensor<64xf32> |
| 120 | + %3 = linalg.fill ins(%cst_0 : f32) outs(%2 : tensor<64xf32>) -> tensor<64xf32> |
| 121 | +
|
| 122 | + // 1. Max reduction (unchanged) |
| 123 | + %4 = linalg.generic { |
| 124 | + indexing_maps = [affine_map<(d0, d1) -> (d0, d1)>, affine_map<(d0, d1) -> (d0)>], |
| 125 | + iterator_types = ["parallel", "reduction"] |
| 126 | + } ins(%0 : tensor<64x1024xf32>) outs(%3 : tensor<64xf32>) { |
| 127 | + ^bb0(%in: f32, %out: f32): |
| 128 | + %8 = arith.maxnumf %in, %out : f32 |
| 129 | + linalg.yield %8 : f32 |
| 130 | + } -> tensor<64xf32> |
| 131 | +
|
| 132 | + %5 = linalg.fill ins(%cst : f32) outs(%2 : tensor<64xf32>) -> tensor<64xf32> |
| 133 | +
|
| 134 | + // 2. Fused: exp(x - max) and sum reduction |
| 135 | + // This computes exp on-the-fly during the reduction |
| 136 | + %6 = linalg.generic { |
| 137 | + indexing_maps = [ |
| 138 | + affine_map<(d0, d1) -> (d0, d1)>, |
| 139 | + affine_map<(d0, d1) -> (d0)>, |
| 140 | + affine_map<(d0, d1) -> (d0)> |
| 141 | + ], |
| 142 | + iterator_types = ["parallel", "reduction"] |
| 143 | + } ins(%0, %4 : tensor<64x1024xf32>, tensor<64xf32>) outs(%5 : tensor<64xf32>) { |
| 144 | + ^bb0(%in: f32, %in_1: f32, %out: f32): |
| 145 | + %8 = arith.subf %in, %in_1 : f32 |
| 146 | + %9 = math.exp %8 : f32 |
| 147 | + %10 = arith.addf %9, %out : f32 |
| 148 | + linalg.yield %10 : f32 |
| 149 | + } -> tensor<64xf32> |
| 150 | +
|
| 151 | + // 3. Fused: exp(x - max) and division |
| 152 | + // This computes exp again and immediately divides by sum |
| 153 | + %7 = linalg.generic { |
| 154 | + indexing_maps = [ |
| 155 | + affine_map<(d0, d1) -> (d0, d1)>, |
| 156 | + affine_map<(d0, d1) -> (d0)>, |
| 157 | + affine_map<(d0, d1) -> (d0)>, |
| 158 | + affine_map<(d0, d1) -> (d0, d1)> |
| 159 | + ], |
| 160 | + iterator_types = ["parallel", "parallel"] |
| 161 | + } ins(%0, %4, %6 : tensor<64x1024xf32>, tensor<64xf32>, tensor<64xf32>) outs(%1 : tensor<64x1024xf32>) { |
| 162 | + ^bb0(%in: f32, %in_1: f32, %in_2: f32, %out: f32): |
| 163 | + %8 = arith.subf %in, %in_1 : f32 |
| 164 | + %9 = math.exp %8 : f32 |
| 165 | + %10 = arith.divf %9, %in_2 : f32 |
| 166 | + linalg.yield %10 : f32 |
| 167 | + } -> tensor<64x1024xf32> |
| 168 | +
|
| 169 | + bufferization.materialize_in_destination %7 in restrict writable %arg1 : (tensor<64x1024xf32>, memref<64x1024xf32>) -> () |
| 170 | + return |
| 171 | +} |
| 172 | +``` |
| 173 | + |
| 174 | +--- |
| 175 | + |
| 176 | +## Stage 3: Max-Sum Reduction Fusion (Online Softmax) |
| 177 | + |
| 178 | + |
| 179 | +**What happens:** The max reduction and sum reduction are fused into a single pass over the data using the "online softmax" algorithm. This algorithm maintains running max and running sum values, applying a correction factor when the max changes. |
| 180 | + |
| 181 | +**Algorithm:** |
| 182 | +``` |
| 183 | +For each element x in the row: |
| 184 | + new_max = max(current_max, x) |
| 185 | + correction = exp(old_max - new_max) |
| 186 | + corrected_sum = current_sum * correction |
| 187 | + new_sum = corrected_sum + exp(x - new_max) |
| 188 | +``` |
| 189 | + |
| 190 | +```mlir |
| 191 | +#map = affine_map<(d0, d1) -> (d0, d1)> |
| 192 | +#map1 = affine_map<(d0, d1) -> (d0)> |
| 193 | +
|
| 194 | +func.func @softmax_v2(%arg0: memref<64x1024xf32>, %arg1: memref<64x1024xf32>) { |
| 195 | + %cst = arith.constant 0.000000e+00 : f32 |
| 196 | + %cst_0 = arith.constant 0xFFC00000 : f32 |
| 197 | + %0 = bufferization.to_tensor %arg0 restrict writable : memref<64x1024xf32> to tensor<64x1024xf32> |
| 198 | + %1 = tensor.empty() : tensor<64x1024xf32> |
| 199 | + %2 = tensor.empty() : tensor<64xf32> |
| 200 | + %3 = linalg.fill ins(%cst_0 : f32) outs(%2 : tensor<64xf32>) -> tensor<64xf32> |
| 201 | + %4 = linalg.fill ins(%cst : f32) outs(%2 : tensor<64xf32>) -> tensor<64xf32> |
| 202 | +
|
| 203 | + // Fused max and sum reduction with online correction |
| 204 | + %5:2 = linalg.generic { |
| 205 | + indexing_maps = [#map, #map1, #map1], |
| 206 | + iterator_types = ["parallel", "reduction"] |
| 207 | + } ins(%0 : tensor<64x1024xf32>) outs(%3, %4 : tensor<64xf32>, tensor<64xf32>) { |
| 208 | + ^bb0(%in: f32, %out_max: f32, %out_sum: f32): |
| 209 | + // Update max |
| 210 | + %new_max = arith.maxnumf %in, %out_max : f32 |
| 211 | +
|
| 212 | + // Compute correction factor for sum when max changes |
| 213 | + // correction = exp(old_max - new_max) |
| 214 | + %max_diff = arith.subf %out_max, %new_max : f32 |
| 215 | + %correction = math.exp %max_diff : f32 |
| 216 | + %corrected_sum = arith.mulf %out_sum, %correction : f32 |
| 217 | +
|
| 218 | + // Add new contribution: exp(value - new_max) |
| 219 | + %val_diff = arith.subf %in, %new_max : f32 |
| 220 | + %exp_val = math.exp %val_diff : f32 |
| 221 | + %new_sum = arith.addf %corrected_sum, %exp_val : f32 |
| 222 | +
|
| 223 | + linalg.yield %new_max, %new_sum : f32, f32 |
| 224 | + } -> (tensor<64xf32>, tensor<64xf32>) |
| 225 | +
|
| 226 | + // Final division to compute softmax |
| 227 | + %6 = linalg.generic { |
| 228 | + indexing_maps = [#map, #map1, #map1, #map], |
| 229 | + iterator_types = ["parallel", "parallel"] |
| 230 | + } ins(%0, %5#0, %5#1 : tensor<64x1024xf32>, tensor<64xf32>, tensor<64xf32>) outs(%1 : tensor<64x1024xf32>) { |
| 231 | + ^bb0(%in: f32, %in_max: f32, %in_sum: f32, %out: f32): |
| 232 | + %7 = arith.subf %in, %in_max : f32 |
| 233 | + %8 = math.exp %7 : f32 |
| 234 | + %9 = arith.divf %8, %in_sum : f32 |
| 235 | + linalg.yield %9 : f32 |
| 236 | + } -> tensor<64x1024xf32> |
| 237 | +
|
| 238 | + bufferization.materialize_in_destination %6 in restrict writable %arg1 : (tensor<64x1024xf32>, memref<64x1024xf32>) -> () |
| 239 | + return |
| 240 | +} |
| 241 | +``` |
0 commit comments