@@ -181,6 +181,9 @@ def infer_parameters(
181181 schedule_params = ScheduleParameters ([layer_params ])
182182 schedule_kind = "elemwise"
183183 elif len (elemwise ) > 0 and len (reduction ) > 0 and len (batch_matmuls ) == 0 :
184+ # Elementwise + reduction kernel
185+ iter_space = reduction [0 ]["iterators" ]
186+ n_reduction_dims = sum (s == "reduction" for s in iter_space )
184187 # elemwise + reduction kernel, e.g. softmax or layer norm
185188 shape = elemwise [- 1 ]["shape" ]
186189 res_elemtype = elemwise [- 1 ]["elemtype" ]
@@ -189,25 +192,46 @@ def infer_parameters(
189192 res_bytes = elemtype_bytes [res_elemtype ]
190193 read_bytes = int (np .prod (shape )) * res_bytes
191194 write_bytes = int (np .prod (shape )) * res_bytes
192-
193- layer_params = {
194- "layer_kind" : "reduction" ,
195- "wg_rows" : 64 ,
196- "sg_rows" : 8 ,
197- "subgroup_size" : 16 ,
198- "reduction_step_size" : 32 ,
199- }
195+ if len (shape ) == 2 :
196+ # 2d softmax like kernel
197+ layer_params = {
198+ "layer_kind" : "reduction" ,
199+ "wg_tile" : [64 , 0 ],
200+ "sg_tile" : [8 , 0 ],
201+ "subgroup_size" : 16 ,
202+ "reduction_tile" : [0 , 32 ],
203+ }
204+
205+ elif len (shape ) == 4 and iter_space [1 ] == "reduction" and n_reduction_dims == 1 :
206+ # 4d rms norm like kernel
207+ layer_params = {
208+ "layer_kind" : "reduction" ,
209+ "sizes" : shape ,
210+ "wg_tile" : [1 , 0 , 128 , 128 ],
211+ "wg_subtile" : [0 , 0 , 16 , 32 ],
212+ "sg_tile" : [0 , 2 , 16 , 32 ],
213+ "reduction_tile" : [0 , 32 , 0 , 0 ],
214+ "subgroup_size" : 16 ,
215+ }
200216
201217 # Ensure shape is divisible by tile sizes.
202218 # Padding or remainder handling is not implemented yet.
203- if shape [0 ] % layer_params ["wg_rows" ] != 0 :
204- raise ValueError (
205- f"Shape { shape } dimension 0 not divisible by wg_rows={ layer_params ['wg_rows' ]} "
206- )
207- if shape [1 ] % layer_params ["reduction_step_size" ] != 0 :
208- raise ValueError (
209- f"Shape { shape } dimension 1 not divisible by reduction_step_size={ layer_params ['reduction_step_size' ]} "
210- )
219+ wg_sizes = (
220+ layer_params ["wg_subtile" ]
221+ if "wg_subtile" in layer_params
222+ else layer_params ["wg_tile" ]
223+ )
224+ reduction_tile = layer_params ["reduction_tile" ]
225+ for i , (size , wg , red ) in enumerate (zip (shape , wg_sizes , reduction_tile )):
226+ if wg > 0 and size % wg != 0 :
227+ raise ValueError (
228+ f"Shape { shape } dimension { i } not divisible by wg_tile={ wg_sizes } "
229+ )
230+ if red > 0 and size % red != 0 :
231+ raise ValueError (
232+ f"Shape { shape } dimension { i } not divisible by reduction_tile={ reduction_tile } "
233+ )
234+
211235 schedule_params = ScheduleParameters ([layer_params ])
212236 schedule_kind = "reduction"
213237 elif len (elemwise ) > 0 and len (reduction ) > 0 and len (batch_matmuls ) > 0 :
@@ -609,7 +633,7 @@ def lower_and_execute_benchmark(
609633 params_cache_json = f"kb_params_level{ level } -{ id } .json" ,
610634 dump_parameters = dump_parameters ,
611635 enable_tuning = execute ,
612- verbose = 2 ,
636+ verbose = verbose ,
613637 )
614638 backend = gpu_backend (
615639 fn_compile ,
@@ -647,13 +671,20 @@ def lower_and_execute_benchmark(
647671 layer .get ("kind" ) == "batch_matmul"
648672 for layer in kernel_metadata .get ("layers" , [])
649673 )
674+ is_reduction = any (
675+ layer .get ("kind" ) == "reduction"
676+ for layer in kernel_metadata .get ("layers" , [])
677+ )
650678 atol = abs (result_ref ).max () * 1e-3
651679 rtol = 1e-3
652680 if result_ref .dtype == torch .bfloat16 :
653681 rtol = 2e-2
654682 if is_attention :
655- # Attention fuses 2 matmuls and a softmax, needs a looser atol .
683+ # Attention fuses 2 matmuls and a softmax.
656684 atol = 2e-2
685+ elif is_reduction :
686+ # RMS norm currently uses bf16 accumulator.
687+ atol = 6e-2
657688 elif is_mlp :
658689 atol = 7e-3
659690 success = torch .allclose (result , result_ref , rtol = rtol , atol = atol )
0 commit comments