Repository navigation
[xegpu] Subgraph-based WG tiling and torch llama3.2 example - #315
Conversation
b936e3d to
b37953c
Compare
|
For reference, here's There are three kernels:
IR dump#map = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1)>
#map2 = affine_map<(d0, d1) -> (d0, d1)>
#map3 = affine_map<(d0, d1, d2) -> (d2)>
#map4 = affine_map<(d0, d1) -> (d1, d0)>
#map5 = affine_map<(d0, d1, d2, d3) -> (d0, d2, d1, d3)>
#map6 = affine_map<(d0, d1, d2, d3) -> (d0, d1, d2, d3)>
#map7 = affine_map<(d0, d1, d2, d3) -> (d2, d3)>
module {
func.func private @main(%arg0: tensor<4x512x2048xbf16>, %arg1: tensor<2048xbf16>, %arg2: tensor<512x2048xbf16>, %arg3: tensor<512x64xbf16>, %arg4: tensor<512x64xbf16>, %arg5: memref<4x8x512x64xbf16>
) attributes {llvm.emit_c_interface} {
%c2_i64 = arith.constant 2 : i64
%cst = arith.constant 0.000000e+00 : f32
%cst_0 = arith.constant 1.000000e-05 : f64
%cst_1 = arith.constant 2.048000e+03 : f32
%0 = tensor.empty() : tensor<4x512x2048xf32>
%1 = linalg.generic {indexing_maps = [#map, #map], iterator_types = ["parallel", "parallel", "parallel"]} ins(%arg0 : tensor<4x512x2048xbf16>) outs(%0 : tensor<4x512x2048xf32>) attrs = {transform
_ext.subgraph_id = 1 : i64, transform_ext.subgraph_kind = "reduction"} {
^bb0(%in: bf16, %out: f32):
%26 = arith.extf %in : bf16 to f32
linalg.yield %26 : f32
} -> tensor<4x512x2048xf32>
%2 = linalg.generic {indexing_maps = [#map, #map], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<4x512x2048xf32>) outs(%0 : tensor<4x512x2048xf32>) attrs = {transform_ext
.subgraph_id = 1 : i64, transform_ext.subgraph_kind = "reduction"} {
^bb0(%in: f32, %out: f32):
%26 = math.fpowi %in, %c2_i64 : f32, i64
linalg.yield %26 : f32
} -> tensor<4x512x2048xf32>
%3 = tensor.empty() : tensor<4x512xf32>
%4 = linalg.fill {transform_ext.subgraph_id = 1 : i64, transform_ext.subgraph_kind = "reduction"} ins(%cst : f32) outs(%3 : tensor<4x512xf32>) -> tensor<4x512xf32>
%5 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "reduction"]} ins(%2 : tensor<4x512x2048xf32>) outs(%4 : tensor<4x512xf32>) attrs = {transform_ext.su
bgraph_id = 1 : i64, transform_ext.subgraph_kind = "reduction"} {
^bb0(%in: f32, %out: f32):
%26 = arith.addf %in, %out : f32
linalg.yield %26 : f32
} -> tensor<4x512xf32>
%6 = linalg.generic {indexing_maps = [#map2, #map2], iterator_types = ["parallel", "parallel"]} ins(%5 : tensor<4x512xf32>) outs(%3 : tensor<4x512xf32>) attrs = {transform_ext.subgraph_id = 1 : i
64, transform_ext.subgraph_kind = "reduction"} {
^bb0(%in: f32, %out: f32):
%26 = arith.divf %in, %cst_1 : f32
linalg.yield %26 : f32
} -> tensor<4x512xf32>
%7 = linalg.generic {indexing_maps = [#map2, #map2], iterator_types = ["parallel", "parallel"]} ins(%6 : tensor<4x512xf32>) outs(%3 : tensor<4x512xf32>) attrs = {transform_ext.subgraph_id = 1 : i
64, transform_ext.subgraph_kind = "reduction"} {
^bb0(%in: f32, %out: f32):
%26 = arith.truncf %cst_0 : f64 to f32
%27 = arith.addf %in, %26 : f32
linalg.yield %27 : f32
} -> tensor<4x512xf32>
%8 = linalg.generic {indexing_maps = [#map2, #map2], iterator_types = ["parallel", "parallel"]} ins(%7 : tensor<4x512xf32>) outs(%3 : tensor<4x512xf32>) attrs = {transform_ext.subgraph_id = 1 : i
64, transform_ext.subgraph_kind = "reduction"} {
^bb0(%in: f32, %out: f32):
%26 = math.rsqrt %in : f32
linalg.yield %26 : f32
} -> tensor<4x512xf32>
%9 = linalg.generic {indexing_maps = [#map, #map1, #map], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1, %8 : tensor<4x512x2048xf32>, tensor<4x512xf32>) outs(%0 : tensor<4x512x2048xf32>) attrs = {transform_ext.subgraph_id = 1 : i64, transform_ext.subgraph_kind = "reduction"} {
^bb0(%in: f32, %in_3: f32, %out: f32):
%26 = arith.mulf %in, %in_3 : f32
linalg.yield %26 : f32
} -> tensor<4x512x2048xf32>
%10 = tensor.empty() : tensor<4x512x2048xbf16>
%11 = linalg.generic {indexing_maps = [#map, #map], iterator_types = ["parallel", "parallel", "parallel"]} ins(%9 : tensor<4x512x2048xf32>) outs(%10 : tensor<4x512x2048xbf16>) attrs = {transform_ext.subgraph_id = 1 : i64, transform_ext.subgraph_kind = "reduction"} {
^bb0(%in: f32, %out: bf16):
%26 = arith.truncf %in : f32 to bf16
linalg.yield %26 : bf16
} -> tensor<4x512x2048xbf16>
%12 = linalg.generic {indexing_maps = [#map3, #map, #map], iterator_types = ["parallel", "parallel", "parallel"]} ins(%arg1, %11 : tensor<2048xbf16>, tensor<4x512x2048xbf16>) outs(%10 : tensor<4x512x2048xbf16>) attrs = {transform_ext.subgraph_anchor, transform_ext.subgraph_id = 1 : i64, transform_ext.subgraph_kind = "reduction"} {
^bb0(%in: bf16, %in_3: bf16, %out: bf16):
%26 = arith.mulf %in, %in_3 : bf16
linalg.yield %26 : bf16
} -> tensor<4x512x2048xbf16>
%13 = tensor.empty() : tensor<2048x512xbf16>
%14 = linalg.generic {indexing_maps = [#map4, #map2], iterator_types = ["parallel", "parallel"]} ins(%arg2 : tensor<512x2048xbf16>) outs(%13 : tensor<2048x512xbf16>) attrs = {transform_ext.subgraph_id = 2 : i64, transform_ext.subgraph_kind = "gemm"} {
^bb0(%in: bf16, %out: bf16):
linalg.yield %in : bf16
} -> tensor<2048x512xbf16>
%collapsed = tensor.collapse_shape %12 [[0, 1], [2]] : tensor<4x512x2048xbf16> into tensor<2048x2048xbf16>
%15 = tensor.empty() : tensor<2048x512xf32>
%16 = linalg.fill {transform_ext.subgraph_id = 2 : i64, transform_ext.subgraph_kind = "gemm"} ins(%cst : f32) outs(%15 : tensor<2048x512xf32>) -> tensor<2048x512xf32>
%17 = linalg.matmul {transform_ext.subgraph_id = 2 : i64, transform_ext.subgraph_kind = "gemm"} ins(%collapsed, %14 : tensor<2048x2048xbf16>, tensor<2048x512xbf16>) outs(%16 : tensor<2048x512xf32>) -> tensor<2048x512xf32>
%18 = linalg.generic {indexing_maps = [#map2, #map2], iterator_types = ["parallel", "parallel"]} ins(%17 : tensor<2048x512xf32>) outs(%13 : tensor<2048x512xbf16>) attrs = {transform_ext.subgraph_anchor, transform_ext.subgraph_id = 2 : i64, transform_ext.subgraph_kind = "gemm"} {
^bb0(%in: f32, %out: bf16):
%26 = arith.truncf %in : f32 to bf16
linalg.yield %26 : bf16
} -> tensor<2048x512xbf16>
%expanded = tensor.expand_shape %18 [[0, 1], [2, 3]] output_shape [4, 512, 8, 64] {transform_ext.subgraph_id = 3 : i64, transform_ext.subgraph_kind = "elementwise"} : tensor<2048x512xbf16> into tensor<4x512x8x64xbf16>
%19 = tensor.empty() : tensor<4x8x512x64xbf16>
%20 = linalg.generic {indexing_maps = [#map5, #map6], iterator_types = ["parallel", "parallel", "parallel", "parallel"]} ins(%expanded : tensor<4x512x8x64xbf16>) outs(%19 : tensor<4x8x512x64xbf16>) attrs = {transform_ext.subgraph_anchor, transform_ext.subgraph_id = 3 : i64, transform_ext.subgraph_kind = "elementwise"} {
^bb0(%in: bf16, %out: bf16):
linalg.yield %in : bf16
} -> tensor<4x8x512x64xbf16>
%21 = linalg.generic {indexing_maps = [#map6, #map7, #map6], iterator_types = ["parallel", "parallel", "parallel", "parallel"]} ins(%20, %arg3 : tensor<4x8x512x64xbf16>, tensor<512x64xbf16>) outs(%19 : tensor<4x8x512x64xbf16>) attrs = {transform_ext.subgraph_id = 3 : i64, transform_ext.subgraph_kind = "elementwise"} {
^bb0(%in: bf16, %in_3: bf16, %out: bf16):
%26 = arith.mulf %in, %in_3 : bf16
linalg.yield %26 : bf16
} -> tensor<4x8x512x64xbf16>
%extracted_slice = tensor.extract_slice %20[0, 0, 0, 0] [4, 8, 512, 32] [1, 1, 1, 1] {transform_ext.subgraph_id = 3 : i64, transform_ext.subgraph_kind = "elementwise"} : tensor<4x8x512x64xbf16> to tensor<4x8x512x32xbf16>
%extracted_slice_2 = tensor.extract_slice %20[0, 0, 0, 32] [4, 8, 512, 32] [1, 1, 1, 1] {transform_ext.subgraph_id = 3 : i64, transform_ext.subgraph_kind = "elementwise"} : tensor<4x8x512x64xbf16> to tensor<4x8x512x32xbf16>
%22 = tensor.empty() : tensor<4x8x512x32xbf16>
%23 = linalg.generic {indexing_maps = [#map6, #map6], iterator_types = ["parallel", "parallel", "parallel", "parallel"]} ins(%extracted_slice_2 : tensor<4x8x512x32xbf16>) outs(%22 : tensor<4x8x512x32xbf16>) attrs = {transform_ext.subgraph_id = 3 : i64, transform_ext.subgraph_kind = "elementwise"} {
^bb0(%in: bf16, %out: bf16):
%26 = arith.negf %in : bf16
linalg.yield %26 : bf16
} -> tensor<4x8x512x32xbf16>
%concat = tensor.concat dim(3) %23, %extracted_slice {transform_ext.subgraph_id = 3 : i64, transform_ext.subgraph_kind = "elementwise"} : (tensor<4x8x512x32xbf16>, tensor<4x8x512x32xbf16>) -> tensor<4x8x512x64xbf16>
%24 = linalg.generic {indexing_maps = [#map6, #map7, #map6], iterator_types = ["parallel", "parallel", "parallel", "parallel"]} ins(%concat, %arg4 : tensor<4x8x512x64xbf16>, tensor<512x64xbf16>) outs(%19 : tensor<4x8x512x64xbf16>) attrs = {transform_ext.subgraph_id = 3 : i64, transform_ext.subgraph_kind = "elementwise"} {
^bb0(%in: bf16, %in_3: bf16, %out: bf16):
%26 = arith.mulf %in, %in_3 : bf16
linalg.yield %26 : bf16
} -> tensor<4x8x512x64xbf16>
%25 = linalg.generic {indexing_maps = [#map6, #map6, #map6], iterator_types = ["parallel", "parallel", "parallel", "parallel"]} ins(%21, %24 : tensor<4x8x512x64xbf16>, tensor<4x8x512x64xbf16>) outs(%19 : tensor<4x8x512x64xbf16>) attrs = {transform_ext.subgraph_id = 3 : i64, transform_ext.subgraph_kind = "elementwise"} {
^bb0(%in: bf16, %in_3: bf16, %out: bf16):
%26 = arith.addf %in, %in_3 : bf16
linalg.yield %26 : bf16
} -> tensor<4x8x512x64xbf16>
bufferization.materialize_in_destination %25 in restrict writable %arg5 : (tensor<4x8x512x64xbf16>, memref<4x8x512x64xbf16>) -> ()
return
}
} |
b37953c to
1363dbb
Compare
|
Overall the proposed flow makes sense to me. I only had a high-level look, it's a bit too large to review everything in detail. Btw, what happens when you run across subgraph that doesn't match anything? Fallback to some defaults, silent error? |
|
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This is a temporary workaround to llvm spir-v backend bug.
- apply fold_tensor_vector_transfers - apply fold_log_of_bool
- support collapsed batch dim case - stop tracing at computations and rank changes
- gemm path uses infer_gemm_store_layout - use annotate_epilogue_broadcasts op - support elementwise kernels
- add --identity-weights and --power-of-two-inputs for llama3 verification - raises an error if KernelBench submodule is not found
1363dbb to
7d926f6
Compare
|
Once causal attention lowering is fixed in torch-mlir (see llvm/torch-mlir#4868), these temporary transform ops can be removed:
|
There was a problem hiding this comment.
🟡 Changes recommended
Conflicting load-layout annotations prevent valid elementwise kernels from lowering, while repeated traversal of shared producers can make compilation exponentially expensive.
2 open issues
What changed in this PR
This PR extends XeGPU lowering to schedule subgraphs independently, supporting composed workloads such as a Llama 3.2 1B prefill decoder.
Changes:
- Adds subgraph preparation, tagging, matching, fusion, and elementwise tiling support.
- Extends GPU launch mapping, layout inference, and tensor/vector rewrites.
- Adds Llama benchmarks, transform regression tests, and verification utilities; corrects MLP accumulation.
| File | Description |
|---|---|
test/transform/test_tag_subgraphs.py |
Tests tagging, scope boundaries, and handle preservation. |
test/transform/test_split_concat_output.py |
Tests concat splitting and numerical equivalence. |
test/transform/test_rewrite_attention_row_mask.py |
Adds caller-managed dead-value cleanup. |
test/transform/test_infer_xegpu_gemm_params.py |
Tests inference across head-merging reshapes. |
test/transform/test_infer_xegpu_elementwise_params.py |
Tests elementwise tiles and unsupported shapes. |
test/transform/test_infer_gemm_store_layout.py |
Tests epilogue ownership and store layouts. |
test/transform/test_get_subgraph_ops.py |
Tests producer and consumer handles. |
test/transform/test_fuse_elementwise_op.py |
Tests scoped, repeatable region fusion. |
test/transform/test_fold_tensor_vector_transfers.py |
Tests transfer folding legality and numerics. |
test/transform/test_fold_reshape_ops_by_expansion.py |
Adds transpose and metadata regression coverage. |
test/transform/test_fold_log_of_bool.py |
Tests boolean-log folding and exact mask values. |
test/transform/test_expand_bool_to_float.py |
Tests boolean-cast expansion. |
test/transform/test_collapse_parallel.py |
Tests iteration coverage and GPU grid mapping. |
test/transform/test_clear_subgraph_tags.py |
Tests scheduling-tag removal. |
test/transform/test_annotate_epilogue_broadcasts.py |
Tests store-derived broadcast layouts. |
test/transform/test_analyze_matmul.py |
Covers expanded global operand dimensions. |
lighthouse/schedule/xegpu/xegpu_to_binary.py |
Expands boolean casts before binary lowering. |
lighthouse/schedule/xegpu/outline_gpu_func_schedule.py |
Configures threads independently by kernel kind. |
lighthouse/schedule/xegpu/matmul_params.json |
Adds GEMM tuning configurations. |
lighthouse/schedule/xegpu/lowering_common.py |
Integrates transfer folding and parallel-axis collapse. |
lighthouse/schedule/xegpu/attention_fusion.py |
Bounds producer tracing and deduplicates fusion handles. |
lighthouse/schedule/xegpu/annotate_layouts_schedule.py |
Adds epilogue and elementwise layout handling. |
lighthouse/ingress/mlir_gen/gpu_mlp_payload.py |
Restricts supplied accumulation to the final layer. |
lighthouse/dialects/transform/transform_ext/utils/matmul_analysis.py |
Handles expanded dimensions and limits producer tracing. |
lighthouse/dialects/transform/transform_ext/ops/trace_producers.py |
Adds scoped tracing and traversal ordering. |
lighthouse/dialects/transform/transform_ext/ops/tag_subgraphs.py |
Adds annotation-only subgraph partitioning. |
lighthouse/dialects/transform/transform_ext/ops/split_concat_output.py |
Splits concat-dependent output tiles. |
lighthouse/dialects/transform/transform_ext/ops/prepare_subgraphs.py |
Normalizes casts and isolates shared producers. |
lighthouse/dialects/transform/transform_ext/ops/match_subgraph.py |
Matches tagged members by ID and role. |
lighthouse/dialects/transform/transform_ext/ops/infer_xegpu_reduction_params.py |
Supports higher-rank trailing reductions. |
lighthouse/dialects/transform/transform_ext/ops/infer_xegpu_elementwise_params.py |
Infers elementwise workgroup and subgroup tiles. |
lighthouse/dialects/transform/transform_ext/ops/infer_gemm_store_layout.py |
Derives store layouts through GEMM epilogues. |
lighthouse/dialects/transform/transform_ext/ops/get_subgraph_ops.py |
Retrieves subgraph producers and consumers. |
lighthouse/dialects/transform/transform_ext/ops/fuse_elementwise_op.py |
Adds region-scoped elementwise fusion. |
lighthouse/dialects/transform/transform_ext/ops/fold_tensor_vector_transfers.py |
Removes intermediate tensor/vector round trips. |
lighthouse/dialects/transform/transform_ext/ops/fold_scalar_or_splat_constant.py |
Accepts workgroup-loop targets. |
lighthouse/dialects/transform/transform_ext/ops/fold_reshape_ops_by_expansion.py |
Extends reshape folding to named transposes. |
lighthouse/dialects/transform/transform_ext/ops/fold_log_of_bool.py |
Replaces boolean logarithms with selects. |
lighthouse/dialects/transform/transform_ext/ops/expand_bool_to_float.py |
Replaces unsigned boolean casts with selects. |
lighthouse/dialects/transform/transform_ext/ops/collapse_parallel.py |
Collapses excess parallel grid dimensions. |
lighthouse/dialects/transform/transform_ext/ops/clear_subgraph_tags.py |
Removes subgraph scheduling annotations. |
lighthouse/dialects/transform/transform_ext/ops/annotate_epilogue_broadcasts.py |
Derives broadcast-input layouts from stores. |
lighthouse/dialects/transform/transform_ext/__init__.py |
Exports the new transform helpers. |
examples/xegpu/mlp.py |
Updates accumulation verification and regression checks. |
examples/xegpu/matmul.py |
Strengthens epilogue and allocation checks. |
examples/xegpu/llama-bench_1B_swiglu.py |
Adds a standalone SwiGLU benchmark. |
examples/xegpu/llama-bench_1B_rmsnorm_swiglu_residual.py |
Adds normalized feed-forward and residual coverage. |
examples/xegpu/llama-bench_1B_rmsnorm_qkv_rope_attention.py |
Combines normalization, projections, RoPE, and attention. |
examples/xegpu/llama-bench_1B_rmsnorm_k_rope.py |
Adds normalized K-projection and RoPE coverage. |
examples/xegpu/llama-bench_1B_k_rope.py |
Adds K-projection and RoPE coverage. |
examples/xegpu/llama-bench_1B_decoder.py |
Adds the complete prefill decoder-layer benchmark. |
examples/xegpu/llama-bench_1B_attention.py |
Adds explicit causal masking and grid checks. |
examples/xegpu/llama-bench_1B_attention_block.py |
Adds attention with output projection and residual. |
examples/xegpu/llama_rope.py |
Provides shared rotary-embedding helpers. |
examples/xegpu/lit.local.cfg |
Enables attention testing and excludes the helper module. |
examples/xegpu/kernel_bench.py |
Supports mixed graphs, verification controls, and custom cache names. |
Review effort: Balanced
This op now annotates matching conventional load ops as well.
Avoids revisiting shared producers multiple times.
| # CHECK: arith.cmpi sge | ||
| # CHECK-NOT: math.log | ||
| # CHECK: xegpu.dpas | ||
| # CHECK: math.exp | ||
| # CHECK: xegpu.dpas | ||
| # CHECK: xegpu.store_nd | ||
| # CHECK: gpu.func @main_kernel | ||
| # CHECK: xegpu.dpas | ||
| # CHECK: arith.addf |
There was a problem hiding this comment.
consider using CHECK-NEXT on all examples.
| h = x.float() | ||
| variance = h.pow(2).mean(dim=-1, keepdim=True) | ||
| h = self.norm_weight * (h * torch.rsqrt(variance + 1e-5)).to(x.dtype) | ||
| return x + self.down_proj(F.silu(self.gate_proj(h)) * self.up_proj(h)) |
There was a problem hiding this comment.
its posisble to add horizontal GEMM fusion for swiglu kernel. consider as a future improvement.
| from lighthouse.dialects.transform.transform_ext import TransformExtensionDialect | ||
|
|
||
|
|
||
| def _plan(store): |
There was a problem hiding this comment.
consider avoiding function, class names like *plan and give a meaningful name.
There was a problem hiding this comment.
renamed all _plan helper functions
| return annotations | ||
|
|
||
|
|
||
| class AnnotateEpilogueLoadsOp( |
There was a problem hiding this comment.
would it be better to consider a generic layout propagation op for this? seems like this is too specific to your workload.
would be nicer to think about how to generalize this to other cases.
There was a problem hiding this comment.
I agree, this could be formulated as a generic store_nd -> load_nd layout propagation method.
| return collapsed.operation | ||
|
|
||
|
|
||
| class CollapseParallelOp(TransformExtensionDialect.Operation, name="collapse_parallel"): |
There was a problem hiding this comment.
this is clearly something upstreamable? I guess forall to gpu should handle this.
There was a problem hiding this comment.
Yes, this is probably generic enough to be upstreamed. Would need to think how to integrate with gpu-map-parallel-loops and convert-parallel-loops-to-gpu passes.
| ): | ||
| """Replace scalar/vector unsigned i1-to-float casts with floating selects. | ||
|
|
||
| LLVM's SPIR-V instruction selector can crash on vector boolean uitofp op. |
There was a problem hiding this comment.
any known perf regression due to this?
There was a problem hiding this comment.
Not that I know of. The tiles are in memory so presumably the impact is small. That said, once torch-mlir can properly handle SDPA causal mask, this conversion should not be needed anymore.
| After vectorization, a vector may be written into a tensor, reshaped, | ||
| sliced or concatenated with other tensors, then read back into a vector. | ||
| This op forwards the vector values through those tensor operations using | ||
| vector shape casts and strided-slice extraction/insertion. It removes |
There was a problem hiding this comment.
there are some developments in mem2reg to support these. worth taking a look.
| class FuseForallEpilogueOp( | ||
| TransformExtensionDialect.Operation, name="fuse_forall_epilogue" | ||
| ): | ||
| """Move operations consuming a tiled tensor into its producer scf.forall. |
There was a problem hiding this comment.
why consumer fusion can not handle this?
There was a problem hiding this comment.
Existing upstream methods like FuseOp and fuse_into_containing op fuse producers. In this case we want to produce the consumers into an existing scf.forall loop. This fusion methods also supports more complex shape-changing epilogue DAGs with reshapes/transposes/slices/concat allowing, e.g., to fuse the RoPE embeddings DAG. We could plausibly use or extend the existing scf::tileAndFuseConsumer or scf::tileAndFuseConsumerOfSlices patterns but they are not currently exposed as transform ops.
| return infer(stored) | ||
|
|
||
|
|
||
| class InferGemmStoreLayoutOp( |
There was a problem hiding this comment.
I think these can be consolidated into a layout propagation op.
There was a problem hiding this comment.
Possibly. I've kept the inference phase and layout propagation separate for now. I think inference part requires payload specific logic whereas the propagation can be more generic traversal method.


Adds subgraph-based WG lowering and torch llama3.2 1B model example.
The
wg_tiling_schedulenow uses subgraph matching to handle large graphs more flexibly. First, subgraphs (like attention, reduction, and gemm+epilogue layers) are matched and tagged with MLIR attributes. The attributes consist of an unique subgraph id (int) and kind:attrs = {transform_ext.subgraph_id = 8 : i64, transform_ext.subgraph_kind = "gemm"}.A single layer can consist of multiple ops, e.g., a
linalg.matmulfollowed bylinalg.genericelementwise epilogue. Both of these ops get the same attributes to indicate that they belong to the same subgraph.The lowering of the subgraph depends on the respective lowering path determined by the
subgraph_kindattribute. In the gemm case, all elementwise epilogue ops are fused into the matmul op. The remaining untagged elementwiselinalg.genericops are tagged as elementwise kernels. All subgraphs will be WG tiled independently and will eventually be lowered to a separate GPU kernel (at WG level they're enclosed in ascf.forallloop which will be lowered to agpu.funccall later on).Once WG tiling has been applied, the individual subgraphs can be identified by the encompassing
scf.forallorgpu.funcops and the required analysis in subsequent lowering schedules is much simpler.Using this functionality this PR adds a torch Llama3.2 1B decoder model example. The model corresponds to prefill phase with causal masking. In fact, a cascade of benchmarks with varying complexity is added for testing the pieces of the llama decoder model:
These kernels can be lowered and executed using the
kernel_bench.pyutility:To enable this capability a number of new transform ops were added. (* indicates ops that are a temporary workaround to a known issue or have a clear upstream implementation plan):
fold_log_of_bool(*): replace boolean-to-float logarithms with zero/negative-infinity selects, eliminating the torch-mlir causal-mask workaround’s logarithm.collapse_parallel(*): collapse leadingscf.paralleldimensions to fit the GPU grid without serializing independent tiles.prepare_subgraphs: normalize projection casts and clone shared producers selected for recomputation before tagging.match_subgraph: retrieve tagged subgraph members by stable ID, optionally filtering by operation name and role.expand_bool_to_float(*): replace unsigned boolean-to-float casts with selects to avoid an LLVM SPIR-V backend crash.split_concat_output(*): split concat-dependent output tiles into disjoint slices to avoid concatenated vectors and register shuffles.clear_subgraph_tags: remove subgraph scheduling annotations.get_subgraph_ops: retrieve a subgraph anchor’s producers and consumers in program order.tag_subgraphs: partition operations into attention, reduction, GEMM and elementwise subgraphs using annotation-only tagging.infer_xegpu_elementwise_params: infer workgroup tile sizes for elementwise iteration spaces.annotate_epilogue_loads: Assign store-derived layouts to ordinary and broadcast epilogue load ops.infer_gemm_store_layout: propagate DPAS accumulator ownership through supported epilogues to infer store layouts.fold_tensor_vector_transfers(*): fold vector transfer chains through tensor reshapes, slices and concatenations to avoid temporary buffers.fuse_forall_epilogue: fuse tensor epilogue DAGs intoscf.foralltiles while preserving external users as additional results.Assisted-by: GPT 6.1 Sol/Luna