@@ -275,21 +275,49 @@ def infer_parameters(
275275 write_bytes = int (batch_size * n_head * n_ctx * d_head * nbytes )
276276
277277 assert d_head == 64 , f"d_head must be 64, got { d_head } "
278- layer_params = {
279- "layer_kind" : "attention" ,
280- "batch_size" : batch_size ,
281- "n_head" : n_head ,
282- "n_ctx" : n_ctx ,
283- "d_head" : d_head ,
284- "wg_tile" : [1 , 1 , 128 ],
285- "sg_rows" : 16 ,
286- "subgroup_size" : 16 ,
287- "reduction_tile" : 64 ,
288- "q_load_tile" : [16 , 16 ],
289- "v_load_tile" : [32 , 32 ],
290- "prefetch_tile" : [16 , 32 ],
291- "nb_prefetch" : 1 ,
292- }
278+
279+ max_elemwise_dims = 0
280+ for layer in [s for s in func_metadata ["layers" ] if s ["kind" ] == "elemwise" ]:
281+ max_elemwise_dims = max (max_elemwise_dims , len (layer ["shape" ]))
282+
283+ if max_elemwise_dims == 4 :
284+ # MHA case
285+ layer_params = {
286+ "layer_kind" : "attention" ,
287+ "batch_size" : batch_size ,
288+ "n_head" : n_head ,
289+ "n_ctx" : n_ctx ,
290+ "d_head" : d_head ,
291+ "wg_tile" : [1 , 1 , 128 ],
292+ "sg_rows" : 16 ,
293+ "subgroup_size" : 16 ,
294+ "reduction_tile" : [0 , 0 , 0 , 64 ],
295+ "q_load_tile" : [16 , 16 ],
296+ "v_load_tile" : [32 , 32 ],
297+ "prefetch_tile" : [16 , 32 ],
298+ "nb_prefetch" : 1 ,
299+ }
300+ elif max_elemwise_dims == 5 :
301+ # GQA case
302+ layer_params = {
303+ "layer_kind" : "attention" ,
304+ "batch_size" : batch_size ,
305+ "n_head" : n_head ,
306+ "n_ctx" : n_ctx ,
307+ "d_head" : d_head ,
308+ "wg_tile" : [1 , 1 , 1 , 128 ],
309+ "sg_rows" : 16 ,
310+ "subgroup_size" : 16 ,
311+ "reduction_tile" : [0 , 0 , 0 , 0 , 64 ],
312+ "q_load_tile" : [16 , 16 ],
313+ "v_load_tile" : [32 , 32 ],
314+ "prefetch_tile" : [16 , 32 ],
315+ "nb_prefetch" : 1 ,
316+ }
317+ else :
318+ raise ValueError (
319+ f"Unsupported attention layer dimensionality: { max_elemwise_dims } "
320+ )
293321
294322 schedule_params = ScheduleParameters ([layer_params ])
295323 schedule_kind = "attention"
@@ -947,6 +975,7 @@ def parser_cli_args():
947975 "initial" ,
948976 "cleanup" ,
949977 "tiled" ,
978+ "reduction-tiled" ,
950979 "vectorized" ,
951980 "bufferized" ,
952981 "gpu-outlining" ,
0 commit comments