From c0fce3b097315ea0492eafb23968a17c49f19dec Mon Sep 17 00:00:00 2001 From: Tuomas Karna Date: Fri, 9 Oct 2026 12:59:39 +0300 Subject: [PATCH 01/18] kernel_bench: add -i option to execute custom torch kernels --- examples/xegpu/kernel_bench.py | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/examples/xegpu/kernel_bench.py b/examples/xegpu/kernel_bench.py index 60b550fb..49c0c9ba 100644 --- a/examples/xegpu/kernel_bench.py +++ b/examples/xegpu/kernel_bench.py @@ -918,6 +918,12 @@ def parser_cli_args(): nargs="+", help="IDs of specific benchmarks to execute (default: all)", ) + parser.add_argument( + "-i", + "--input", + nargs="+", + help="Input Python kernel files. Can be used instead of --level and --benchmark to specify custom kernels.", + ) parser.add_argument( "--datatype", type=str, @@ -995,11 +1001,21 @@ def parser_cli_args(): args = parser_cli_args() kb_level = args.level benchmarks = args.benchmark + input_files = args.input + if input_files is not None and kb_level is not None and benchmarks is not None: + raise ValueError("Cannot set both --input, and --level and --benchmark. ") + if input_files is None and kb_level is None and benchmarks is None: + raise ValueError("Either --input or --level and --benchmark must be specified.") + stop_at_stage = args.dump_kernel input_shapes = parse_input_shapes(args.input_shapes) - kb_pattern = f"level{kb_level}/*.py" - bench_list = get_benchmarks(kb_pattern, include=benchmarks) + if input_files is None: + kb_pattern = f"level{kb_level}/*.py" + bench_list = get_benchmarks(kb_pattern, include=benchmarks) + else: + bench_list = [(i + 1, Path(f)) for i, f in enumerate(input_files)] + kb_level = 0 if args.dump_csv and not stop_at_stage: csv_file = "out_kernelbench.csv" @@ -1008,7 +1024,7 @@ def parser_cli_args(): csv_logger = None for bench_id, bench_path in bench_list: - short_path = bench_path.parent.name + "/" + bench_path.name + short_path = Path(*bench_path.parts[-2:]) print("-" * 80) print(f"Executing benchmark: {short_path}", flush=True) entry = { From 5dcd7e20936a5d595c4e49017907c91a6264321e Mon Sep 17 00:00:00 2001 From: Tuomas Karna Date: Tue, 13 Oct 2026 08:58:00 +0300 Subject: [PATCH 02/18] add llama-bench_1B_attention kernel --- examples/xegpu/llama-bench_1B_attention.py | 54 ++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 examples/xegpu/llama-bench_1B_attention.py diff --git a/examples/xegpu/llama-bench_1B_attention.py b/examples/xegpu/llama-bench_1B_attention.py new file mode 100644 index 00000000..e98cee2f --- /dev/null +++ b/examples/xegpu/llama-bench_1B_attention.py @@ -0,0 +1,54 @@ +import torch +import torch.nn as nn + + +class Model(nn.Module): + """ + Llama3.2-1B scaled dot-product attention (GQA + causal), the realistic + counterpart to KernelBench level1-97. + + Differences from the synthetic level1-97 SDPA: + * d_head = 64 (Llama3.2-1B), the per-head reduction dimension, within the + current <= 64 lowering limit. (Llama3-8B uses d_head = 128, which still + needs the new Intel-GPU head tiling.) + * Grouped-query attention: 32 query heads share 8 key/value heads + (num_heads / num_kv_heads = 4). K/V carry fewer heads than Q and are + broadcast across the group (enable_gqa=True). + * Causal masking (is_causal=True), as in autoregressive decoding/prefill. + + RoPE is intentionally NOT applied here; the rotary embedding is a separate + stepping-stone (llama-bench_4) because rotate_half is a slice+concat shuffle + on the innermost (head) dimension rather than a plain matmul/elementwise op. + Q/K/V are taken as already-projected inputs, matching level1-97. + """ + + def __init__(self): + super().__init__() + + def forward( + self, Q: torch.Tensor, K: torch.Tensor, V: torch.Tensor + ) -> torch.Tensor: + # Q: (batch, num_heads, n_ctx, d_head); K/V: (batch, num_kv_heads, n_ctx, d_head) + causal = False # NOTE handle causal masking in the lowering + return torch.nn.functional.scaled_dot_product_attention( + Q, K, V, is_causal=causal, enable_gqa=True + ) + + +# Llama3.2-1B attention config. +batch_size = 4 +n_ctx = 512 +num_heads = 32 # query heads +num_kv_heads = 8 # grouped key/value heads (GQA group size = 4) +d_head = 64 # per-head reduction dim; within the current <= 64 cap + + +def get_inputs(): + Q = torch.rand(batch_size, num_heads, n_ctx, d_head) + K = torch.rand(batch_size, num_kv_heads, n_ctx, d_head) + V = torch.rand(batch_size, num_kv_heads, n_ctx, d_head) + return [Q, K, V] + + +def get_init_inputs(): + return [] From 19d4ae89e6b47339ad5959e666b9fc273041fac4 Mon Sep 17 00:00:00 2001 From: Tuomas Karna Date: Thu, 15 Oct 2026 21:32:25 +0300 Subject: [PATCH 03/18] contraction_normalization: fix for scalar=None case --- .../transform_ext/utils/contraction_normalization.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lighthouse/dialects/transform/transform_ext/utils/contraction_normalization.py b/lighthouse/dialects/transform/transform_ext/utils/contraction_normalization.py index 710be831..0f9367cb 100644 --- a/lighthouse/dialects/transform/transform_ext/utils/contraction_normalization.py +++ b/lighthouse/dialects/transform/transform_ext/utils/contraction_normalization.py @@ -238,9 +238,11 @@ def _trace_producer_scale(contraction, operand_index): body = op.regions[0].blocks[0] args = list(body.arguments)[: len(inputs)] scalar = _producer(body.operations[-1].operands[0]) + if scalar is None: + return None, None # Only constants may accompany the selected scalar operation. if any( - inner.operation != getattr(scalar, "operation", None) + inner.operation != scalar.operation and not isinstance(inner, (arith.ConstantOp, linalg.YieldOp)) for inner in body.operations ): From b45c8e51a0902c376b855489c5887a82ca603cb3 Mon Sep 17 00:00:00 2001 From: Tuomas Karna Date: Thu, 8 Oct 2026 22:35:57 +0300 Subject: [PATCH 04/18] infer_xegpu_attention_params: support GQA; reduction tile is a vector --- .../ops/infer_xegpu_attention_params.py | 35 +++++++++++++------ 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/lighthouse/dialects/transform/transform_ext/ops/infer_xegpu_attention_params.py b/lighthouse/dialects/transform/transform_ext/ops/infer_xegpu_attention_params.py index 8dd6aa2c..4a4a8716 100644 --- a/lighthouse/dialects/transform/transform_ext/ops/infer_xegpu_attention_params.py +++ b/lighthouse/dialects/transform/transform_ext/ops/infer_xegpu_attention_params.py @@ -17,10 +17,7 @@ class InferXeGPUAttentionParamsOp( directly to the tiling routines without splitting; `reduction_tile` is a scalar i64 param. - NOTE: placeholder implementation; the tiles are hard-coded for now and the - analysis deriving them from `target` will be added in a follow-up. Later - schedules will additionally need sg_rows, q/v load tiles and prefetch - parameters; those results will be added here when required. + NOTE: This is just a placeholder implementation with hard-coded tile sizes. Args: target: Handle to the attention anchor op(s). @@ -51,13 +48,31 @@ def apply( results: transform.TransformResults, state: transform.TransformState, ) -> DiagnosedSilenceableFailure: - # TODO: derive the tiles from `target`; placeholders for now. - i64 = ir.IntegerType.get_signless(64) + target_ops = state.get_payload_ops(op.target) + if len(target_ops) == 0: + return DiagnosedSilenceableFailure.SilenceableFailure + rank = ir.ShapedType(target_ops[0].results[0].type).rank + size_attrs = InferXeGPUAttentionParamsOp._size_attrs - results.set_params(op.wg_tile, size_attrs([1, 1, 128])) - results.set_params(op.sg_tile, size_attrs([0, 0, 16])) - # reduction_tile is a scalar; generalize to an array later if needed. - results.set_params(op.reduction_tile, [ir.IntegerAttr.get(i64, 64)]) + # WG/SG row sizes and the reduction (K/V seq) tile are hard-coded. + wg_rows, sg_rows, reduction_tile = 128, 16, 64 + # Tile every leading parallel dim (incl. the GQA group) by 1, the + # query-row dim by the WG row size, and leave d_head untiled. + if rank == 4: # plain MHA: (batch, head, query_row, d_head) + wg, sg = [1, 1, wg_rows], [0, 0, sg_rows] + red = [0, 0, 0, reduction_tile] + elif rank == 5: # GQA: (batch, kv_head, group, query_row, d_head) + wg, sg = [1, 1, 1, wg_rows], [0, 0, 0, sg_rows] + red = [0, 0, 0, 0, reduction_tile] + else: + op.location.emit_error( + "infer_xegpu_attention_params: unsupported attention leaf " + f"rank {rank}; expected 4 (MHA) or 5 (GQA)" + ) + return DiagnosedSilenceableFailure.SilenceableFailure + results.set_params(op.wg_tile, size_attrs(wg)) + results.set_params(op.sg_tile, size_attrs(sg)) + results.set_params(op.reduction_tile, size_attrs(red)) return DiagnosedSilenceableFailure.Success @staticmethod From 390560877f3f6318af467abd26dfc96ebdf1852a Mon Sep 17 00:00:00 2001 From: Tuomas Karna Date: Thu, 15 Oct 2026 22:07:58 +0300 Subject: [PATCH 05/18] attention_fusion: transform_to_flash_attention reduction tile is a vector Leading parallel dimensions must be zero-padded. --- lighthouse/schedule/xegpu/attention_fusion.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lighthouse/schedule/xegpu/attention_fusion.py b/lighthouse/schedule/xegpu/attention_fusion.py index 7ff767a6..ec97894d 100644 --- a/lighthouse/schedule/xegpu/attention_fusion.py +++ b/lighthouse/schedule/xegpu/attention_fusion.py @@ -8,9 +8,7 @@ from lighthouse.dialects.transform import transform_ext -def transform_to_flash_attention( - anytype, func, reduction_tile, n_parallel -) -> ir.Operation: +def transform_to_flash_attention(anytype, func, reduction_tile) -> ir.Operation: """Fold the attention chain into the tiled row-max loop. After generalization: ``s = Q@K^T * scale``, ``m = max(s)``, @@ -19,6 +17,9 @@ def transform_to_flash_attention( Assumes finite V and finite or -inf scores; the output normalization must preserve the fully-masked-row guard. All--inf row masks are rewritten using the running/final maximum before sinking score producers. + + `reduction_tile` is a vector where the leading parallel dimensions are + zero, e.g. `[0, 0, ..., tile_size]`. """ # Match the necessary op handles. Number of reduction and elementwise loops # can vary due to possible prologue/epilogue and P mask generation. @@ -57,8 +58,7 @@ def transform_to_flash_attention( p_op = transform.get_parent_op(anytype, exp_op, op_name="linalg.generic") # Tile max loop, only the final reduction dimension. - sizes = [0] * n_parallel + [reduction_tile] - _, [reduction_loop], _ = lh_transform.tile(max_op, tile_sizes=sizes) + _, [reduction_loop], _ = lh_transform.tile(max_op, tile_sizes=reduction_tile) transform.annotate(reduction_loop, transform_ext.REDUCTION_LOOP_ATTR_NAME) # Fuse the row sum first. P@V still needs the original exp term. From 0ba4306cd219987c998ce01232fdea6318f5f700 Mon Sep 17 00:00:00 2001 From: Tuomas Karna Date: Thu, 15 Oct 2026 22:14:05 +0300 Subject: [PATCH 06/18] wg_tiling_schedule: updates for GQA attention case --- lighthouse/schedule/xegpu/wg_tiling_schedule.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/lighthouse/schedule/xegpu/wg_tiling_schedule.py b/lighthouse/schedule/xegpu/wg_tiling_schedule.py index 4ce9d1d0..0d6ceeff 100644 --- a/lighthouse/schedule/xegpu/wg_tiling_schedule.py +++ b/lighthouse/schedule/xegpu/wg_tiling_schedule.py @@ -2,7 +2,7 @@ from mlir import ir from mlir.dialects import transform -from mlir.dialects.transform import structured +from mlir.dialects.transform import structured, tensor import lighthouse.transform as lh_transform from lighthouse.pipeline.helper import ( apply_registered_pass, @@ -181,8 +181,17 @@ def apply_attention_tiling(func: ir.Operation) -> ir.Operation: """Apply attention-layer workgroup tiling to the given function.""" anytype = transform.AnyOpType.get() - # Temporary Python equivalent of the native reshape-expansion patterns. - func = transform_ext.fold_reshape_ops_by_expansion(func) + # Clean up to fully expanded shapes by eliminating expand/collapse shape pairs. + # Generalize all transpose ops. + transpose_ops = lh_transform.match_op(func, "linalg.transpose") + structured.structured_generalize(anytype, transpose_ops) + for i in range(3): + # Fold reshape ops by expansion and fold extract_slice into tensor.empty + # Repeat until no more folding. + func = transform_ext.fold_reshape_ops_by_expansion(func) + with ir.InsertionPoint(transform.apply_patterns(func).patterns): + tensor.apply_patterns_tensor_fold_tensor_empty(fold_single_use_only=False) + lh_transform.cleanup(func) # Find the subtract and exp reduction loops and fuse them. subf_op = transform_ext.extract_handle( @@ -224,7 +233,7 @@ def apply_attention_tiling(func: ir.Operation) -> ir.Operation: ) lh_transform.cleanup(func) - func = transform_to_flash_attention(anytype, func, reduction_tile, n_parallel=3) + func = transform_to_flash_attention(anytype, func, reduction_tile) # Inline splat-constant generic inputs (e.g. masked-select zero) as # scalars so they are not vectorized into reads of constant tensors. From 57b54118ae28d8e27796694614c851aa62b42a20 Mon Sep 17 00:00:00 2001 From: Tuomas Karna Date: Fri, 9 Oct 2026 09:00:47 +0300 Subject: [PATCH 07/18] transform_ext: add expand_output_destination op --- .../transform/transform_ext/__init__.py | 2 + .../ops/expand_output_destination.py | 103 ++++++++++++++++++ .../test_expand_output_destination.py | 74 +++++++++++++ 3 files changed, 179 insertions(+) create mode 100644 lighthouse/dialects/transform/transform_ext/ops/expand_output_destination.py create mode 100644 test/transform/test_expand_output_destination.py diff --git a/lighthouse/dialects/transform/transform_ext/__init__.py b/lighthouse/dialects/transform/transform_ext/__init__.py index 34a8e391..9937ecd6 100644 --- a/lighthouse/dialects/transform/transform_ext/__init__.py +++ b/lighthouse/dialects/transform/transform_ext/__init__.py @@ -11,6 +11,7 @@ from .ops.convert_func_results_to_args import convert_func_results_to_args from .ops.narrow_vector_contract_operands import narrow_vector_contract_operands from .ops.emit_definite_failure import emit_definite_failure +from .ops.expand_output_destination import expand_output_destination from .ops.extract_handle import extract_handle from .ops.get_tileable_consumers import get_tileable_consumers from .ops.get_tiling_sizes import get_tiling_sizes @@ -54,6 +55,7 @@ "compute_sg_layout", "convert_func_results_to_args", "emit_definite_failure", + "expand_output_destination", "extract_handle", "filter_by_name", "filter_contraction_ops", diff --git a/lighthouse/dialects/transform/transform_ext/ops/expand_output_destination.py b/lighthouse/dialects/transform/transform_ext/ops/expand_output_destination.py new file mode 100644 index 00000000..51fc547f --- /dev/null +++ b/lighthouse/dialects/transform/transform_ext/ops/expand_output_destination.py @@ -0,0 +1,103 @@ +from mlir import ir +from mlir.dialects import ext, transform, memref, bufferization +from mlir.dialects.transform import DiagnosedSilenceableFailure + +from lighthouse.dialects.transform.transform_ext import TransformExtensionDialect + + +class ExpandOutputDestinationOp( + TransformExtensionDialect.Operation, name="expand_output_destination" +): + """Remove a reshape from a result's write path so it bufferizes in place. + + Rewrites ``materialize_in_destination(collapse_shape(X), D)`` into + ``materialize_in_destination(X, memref.expand_shape(D))``. + + WG-tiled attention writes its result in the GQA-expanded N-D layout, but the + output memref is the collapsed (N-1)-D layout, so a ``tensor.collapse_shape`` + sits between the loop result and the destination. That reshape makes the loop + result and the destination differ in shape, so empty-tensor elimination cannot + fold the loop's output into the destination and bufferization materializes a + separate buffer and copies it out. Reshaping the destination up to the loop's + layout removes the reshape from the write path, so elimination fires and the + loop writes the output in place. + + Only rewrites ``materialize_in_destination`` ops whose destination is a memref + and whose source is a single ``tensor.collapse_shape``; a no-op otherwise. + """ + + target: ext.Operand[transform.AnyOpType] + result: ext.Result[transform.AnyOpType[()]] = ext.infer_result() + + @classmethod + def attach_interface_impls(cls, ctx=None): + cls.TransformOpInterfaceModel.attach(cls.OPERATION_NAME, context=ctx) + cls.MemoryEffectsOpInterfaceModel.attach(cls.OPERATION_NAME, context=ctx) + + class TransformOpInterfaceModel(transform.TransformOpInterface): + @staticmethod + def apply(op, _rewriter, results, state) -> DiagnosedSilenceableFailure: + targets = list(state.get_payload_ops(op.target)) + for func in targets: + _rewrite(func) + results.set_ops(op.result, targets) + return DiagnosedSilenceableFailure.Success + + @staticmethod + def allow_repeated_handle_operands(_op) -> bool: + return False + + class MemoryEffectsOpInterfaceModel(ir.MemoryEffectsOpInterface): + @staticmethod + def get_effects(op): + return ( + transform.only_reads_handle(op.op_operands) + + transform.produces_handle(op.results) + + transform.modifies_payload() + ) + + +def _reassociation(collapse: ir.OpView) -> list[list[int]]: + return [ + [ir.IntegerAttr(idx).value for idx in ir.ArrayAttr(group)] + for group in ir.ArrayAttr(collapse.attributes["reassociation"]) + ] + + +def _rewrite(func: ir.OpView) -> None: + materializes = [ + o + for o in func.regions[0].blocks[0].operations + if o.operation.name == "bufferization.materialize_in_destination" + ] + for mat in materializes: + source, dest = mat.operands[0], mat.operands[1] + if not isinstance(dest.type, ir.MemRefType): + continue + collapse = source.owner + if collapse.operation.name != "tensor.collapse_shape": + continue + expanded = collapse.operands[0] + expanded_type = ir.RankedTensorType(expanded.type) + with ir.InsertionPoint(mat): + dest_type = ir.MemRefType.get( + expanded_type.shape, expanded_type.element_type + ) + dest_expanded = memref.expand_shape( + dest_type, + dest, + _reassociation(collapse), + [], + static_output_shape=list(expanded_type.shape), + ) + bufferization.materialize_in_destination( + None, expanded, dest_expanded, restrict=True, writable=True + ) + mat.operation.erase() + + +def expand_output_destination( + target: ir.Value[transform.AnyOpType], +) -> ir.Value[transform.AnyOpType]: + """Rewrite result ``materialize_in_destination`` to write in place.""" + return ExpandOutputDestinationOp(target=target).result diff --git a/test/transform/test_expand_output_destination.py b/test/transform/test_expand_output_destination.py new file mode 100644 index 00000000..45b039f6 --- /dev/null +++ b/test/transform/test_expand_output_destination.py @@ -0,0 +1,74 @@ +# RUN: %PYTHON %s | FileCheck %s + +"""Tests for the expand_output_destination transform op. + +The op rewrites `materialize_in_destination(collapse_shape(X), D_memref)` into +`materialize_in_destination(X, memref.expand_shape(D_memref))`, moving the +reshape off the result write path so the producer can bufferize in place. +""" + +from mlir import ir +from mlir.dialects import transform + +import lighthouse.dialects as lh_dialects +from lighthouse import transform as lh_transform +from lighthouse.dialects.transform import transform_ext +from lighthouse.schedule.builders import schedule_boilerplate + + +def run(name: str, payload_str: str): + print(f"Test: {name}", flush=True) + with ir.Context(), ir.Location.unknown(): + lh_dialects.register_and_load() + payload = ir.Module.parse(payload_str) + with schedule_boilerplate() as (sched, named_seq): + func = lh_transform.match_op(named_seq.bodyTarget, "func.func") + transform_ext.expand_output_destination(func) + transform.yield_() + sched.body.operations[0].apply(payload.operation) + payload.operation.verify() + print(payload) + + +# A 5-D result collapsed to the 4-D output memref: the op expands the destination +# back to 5-D so the materialize writes the un-reshaped result in place. +COLLAPSE = """ +module { + func.func @main(%out: memref<4x32x512x64xbf16>) attributes {llvm.emit_c_interface} { + %0 = tensor.empty() : tensor<4x8x4x512x64xbf16> + %collapsed = tensor.collapse_shape %0 [[0], [1, 2], [3], [4]] + : tensor<4x8x4x512x64xbf16> into tensor<4x32x512x64xbf16> + bufferization.materialize_in_destination %collapsed in restrict writable %out + : (tensor<4x32x512x64xbf16>, memref<4x32x512x64xbf16>) -> () + return + } +} +""" + +# CHECK-LABEL: Test: collapse_to_expand +# The destination memref is expanded to the result's 5-D layout ... +# CHECK: memref.expand_shape +# CHECK-SAME: memref<4x32x512x64xbf16> into memref<4x8x4x512x64xbf16> +# ... and the result is materialized into it without any reshape on the source. +# CHECK: bufferization.materialize_in_destination %0 in restrict writable +# CHECK-SAME: (tensor<4x8x4x512x64xbf16>, memref<4x8x4x512x64xbf16>) +run("collapse_to_expand", COLLAPSE) + + +# No collapse_shape on the write path: the op leaves the materialize unchanged. +NOOP = """ +module { + func.func @main(%out: memref<4x32x512x64xbf16>) attributes {llvm.emit_c_interface} { + %0 = tensor.empty() : tensor<4x32x512x64xbf16> + bufferization.materialize_in_destination %0 in restrict writable %out + : (tensor<4x32x512x64xbf16>, memref<4x32x512x64xbf16>) -> () + return + } +} +""" + +# CHECK-LABEL: Test: noop_no_collapse +# CHECK: bufferization.materialize_in_destination %0 in restrict writable +# CHECK-SAME: (tensor<4x32x512x64xbf16>, memref<4x32x512x64xbf16>) +# CHECK-NOT: memref.expand_shape +run("noop_no_collapse", NOOP) From 88a322230823ecf654c3bdc75a192031a0c61eb7 Mon Sep 17 00:00:00 2001 From: Tuomas Karna Date: Fri, 9 Oct 2026 09:49:31 +0300 Subject: [PATCH 08/18] wg_tiling_schedule: apply expand_output_destination for attention kernel --- lighthouse/schedule/xegpu/wg_tiling_schedule.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lighthouse/schedule/xegpu/wg_tiling_schedule.py b/lighthouse/schedule/xegpu/wg_tiling_schedule.py index 0d6ceeff..624913db 100644 --- a/lighthouse/schedule/xegpu/wg_tiling_schedule.py +++ b/lighthouse/schedule/xegpu/wg_tiling_schedule.py @@ -242,6 +242,12 @@ def apply_attention_tiling(func: ir.Operation) -> ir.Operation: enable_fastmath_optimizations(func) lh_transform.cleanup(func) + # Write the result through an expanded view of the output memref so the GQA + # collapse_shape leaves the write path and bufferization elides the output + # buffer + copy. + func = transform_ext.expand_output_destination(func) + lh_transform.cleanup(func) + return func From 975e6e847abbcd3af5836e5eb6b5e4176dd3c7a5 Mon Sep 17 00:00:00 2001 From: Tuomas Karna Date: Fri, 9 Oct 2026 09:50:58 +0300 Subject: [PATCH 09/18] infer_xegpu_attention_params: support vector.contract inference --- .../ops/infer_xegpu_attention_params.py | 45 +++++++++++++------ 1 file changed, 32 insertions(+), 13 deletions(-) diff --git a/lighthouse/dialects/transform/transform_ext/ops/infer_xegpu_attention_params.py b/lighthouse/dialects/transform/transform_ext/ops/infer_xegpu_attention_params.py index 4a4a8716..eeb6fe94 100644 --- a/lighthouse/dialects/transform/transform_ext/ops/infer_xegpu_attention_params.py +++ b/lighthouse/dialects/transform/transform_ext/ops/infer_xegpu_attention_params.py @@ -1,5 +1,5 @@ from mlir import ir -from mlir.dialects import ext, transform +from mlir.dialects import ext, transform, linalg, vector from mlir.dialects.transform import DiagnosedSilenceableFailure from lighthouse.dialects.transform.transform_ext import TransformExtensionDialect @@ -49,25 +49,44 @@ def apply( state: transform.TransformState, ) -> DiagnosedSilenceableFailure: target_ops = state.get_payload_ops(op.target) - if len(target_ops) == 0: + if len(target_ops) != 1: + # expecting a single anchor op return DiagnosedSilenceableFailure.SilenceableFailure - rank = ir.ShapedType(target_ops[0].results[0].type).rank + target_op = target_ops[0] + rank = ir.ShapedType(target_op.results[0].type).rank size_attrs = InferXeGPUAttentionParamsOp._size_attrs - # WG/SG row sizes and the reduction (K/V seq) tile are hard-coded. + wg_rows, sg_rows, reduction_tile = 128, 16, 64 - # Tile every leading parallel dim (incl. the GQA group) by 1, the - # query-row dim by the WG row size, and leave d_head untiled. - if rank == 4: # plain MHA: (batch, head, query_row, d_head) - wg, sg = [1, 1, wg_rows], [0, 0, sg_rows] - red = [0, 0, 0, reduction_tile] - elif rank == 5: # GQA: (batch, kv_head, group, query_row, d_head) - wg, sg = [1, 1, 1, wg_rows], [0, 0, 0, sg_rows] - red = [0, 0, 0, 0, reduction_tile] + + if isinstance(target_op, linalg.GenericOp): + # Assume non-tiled linalg.matmul + # WG/SG row sizes and the reduction (K/V seq) tile are hard-coded. + # Tile every leading parallel dim (incl. the GQA group) by 1, the + # query-row dim by the WG row size, and leave d_head untiled. + if rank == 4: # plain MHA: (batch, head, query_row, d_head) + wg, sg = [1, 1, wg_rows], [0, 0, sg_rows] + red = [0, 0, 0, reduction_tile] + elif rank == 5: # GQA: (batch, kv_head, group, query_row, d_head) + wg, sg = [1, 1, 1, wg_rows], [0, 0, 0, sg_rows] + red = [0, 0, 0, 0, reduction_tile] + else: + op.location.emit_error( + "infer_xegpu_attention_params: unsupported attention leaf " + f"rank {rank}; expected 4 (MHA) or 5 (GQA)" + ) + return DiagnosedSilenceableFailure.SilenceableFailure + elif isinstance(target_op, vector.ContractionOp) and rank == 2: + # Assume tiled vector.contract, op's result shape is + # (wg_rows, reduction_tile) + wg_rows, _ = ir.ShapedType(target_op.results[0].type).shape + wg = [wg_rows, 0] + sg = [sg_rows, 0] + red = [0, reduction_tile] else: op.location.emit_error( "infer_xegpu_attention_params: unsupported attention leaf " - f"rank {rank}; expected 4 (MHA) or 5 (GQA)" + f"op {target_op.operation.name} with rank {rank}" ) return DiagnosedSilenceableFailure.SilenceableFailure results.set_params(op.wg_tile, size_attrs(wg)) From 58cc9280d918e34bb346deff5c31231376b03fab Mon Sep 17 00:00:00 2001 From: Tuomas Karna Date: Fri, 9 Oct 2026 10:52:20 +0300 Subject: [PATCH 10/18] annotate_layouts_schedule: support llama3 attention kernel --- .../xegpu/annotate_layouts_schedule.py | 36 +++++++++++++------ 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/lighthouse/schedule/xegpu/annotate_layouts_schedule.py b/lighthouse/schedule/xegpu/annotate_layouts_schedule.py index d977e925..1e8f1417 100644 --- a/lighthouse/schedule/xegpu/annotate_layouts_schedule.py +++ b/lighthouse/schedule/xegpu/annotate_layouts_schedule.py @@ -210,9 +210,9 @@ def annotate_reduction(gpu_func: ir.Operation, anchor_op: ir.Operation): def annotate_attention(gpu_func: ir.Operation, anchor_op: ir.Operation): - wg_tile, sg_tile, reduction_tile = transform_ext.infer_xegpu_attention_params( - anchor_op - ) + # wg_tile, sg_tile, reduction_tile = transform_ext.infer_xegpu_attention_params( + # anchor_op + # ) # TODO infer the layout parameters - hardcoded for now # Insert prefetches for the K and V tiles of the reduction loop. Each @@ -226,17 +226,31 @@ def annotate_attention(gpu_func: ir.Operation, anchor_op: ir.Operation): prefetch_sg_layout = [4, 2] prefetch_inst_data = list(prefetch_sg_data) + def _add_prefetch(load_op): + add_prefetch( + load_op, + nb_prefetch, + sg_layout=prefetch_sg_layout, + sg_data=prefetch_sg_data, + inst_data=prefetch_inst_data, + ) + if nb_prefetch > 0: - reduction_loop = lh_transform.match_op(gpu_func, "scf.for") + # Add prefetch ops to loads within the reduction loop. + reduction_loop = transform_ext.extract_handle( + lh_transform.match_op(gpu_func, "scf.for"), 0, silenceable=True + ) load_ops = lh_transform.match_op(reduction_loop, "xegpu.load_nd") with lh_transform.foreach(load_ops) as load_op: - add_prefetch( - load_op, - nb_prefetch, - sg_layout=prefetch_sg_layout, - sg_data=prefetch_sg_data, - inst_data=prefetch_inst_data, - ) + _add_prefetch(load_op) + transform.yield_() + # Handle possible nested scf.for loop + nested_for_ops = lh_transform.match_op(reduction_loop, "scf.for") + with lh_transform.foreach(nested_for_ops) as for_op: + nested_load_ops = lh_transform.match_op(for_op, "xegpu.load_nd") + with lh_transform.foreach(nested_load_ops) as load_op: + _add_prefetch(load_op) + transform.yield_() transform.yield_() lh_transform.cleanup(gpu_func) From 1583fb0c327b00da2a7d75919e8184fbe4311a00 Mon Sep 17 00:00:00 2001 From: Tuomas Karna Date: Fri, 16 Oct 2026 23:24:32 +0300 Subject: [PATCH 11/18] fused_attention_schedule: support GQA case --- .../xegpu/fused_attention_schedule.py | 98 ++++++++++++------- 1 file changed, 62 insertions(+), 36 deletions(-) diff --git a/lighthouse/schedule/xegpu/fused_attention_schedule.py b/lighthouse/schedule/xegpu/fused_attention_schedule.py index e9072977..ba371a66 100644 --- a/lighthouse/schedule/xegpu/fused_attention_schedule.py +++ b/lighthouse/schedule/xegpu/fused_attention_schedule.py @@ -25,6 +25,7 @@ transform_to_flash_attention, enable_fastmath_optimizations, ) +from .annotate_layouts_schedule import add_prefetch def fused_attention_schedule( @@ -155,9 +156,6 @@ def bundle_xegpu_fused_attention_schedule( lh_transform.match_op(func, ["linalg.elementwise", "linalg.batch_matmul"]), ) - # Temporary Python equivalent of the native reshape-expansion patterns. - func = transform_ext.fold_reshape_ops_by_expansion(func) - # Normalize possible singleton dimensions so tile+fuse logic works. with ir.InsertionPoint(transform.apply_patterns(func).patterns): # Drop unused argmax results so online fusion sees a simple row max. @@ -172,6 +170,18 @@ def bundle_xegpu_fused_attention_schedule( tensor.apply_patterns_tensor_fold_tensor_empty(fold_single_use_only=True) lh_transform.cleanup(func) + # Clean up to fully expanded shapes by eliminating expand/collapse shape pairs. + # Generalize all transpose ops. + transpose_ops = lh_transform.match_op(func, "linalg.transpose") + structured.structured_generalize(anytype, transpose_ops) + for i in range(3): + # Fold reshape ops by expansion and fold extract_slice into tensor.empty + # Repeat until no more folding. + func = transform_ext.fold_reshape_ops_by_expansion(func) + with ir.InsertionPoint(transform.apply_patterns(func).patterns): + tensor.apply_patterns_tensor_fold_tensor_empty(fold_single_use_only=False) + lh_transform.cleanup(func) + # Find the subtract and exp reduction loops and fuse them. subf_op = transform_ext.extract_handle( lh_transform.match_op(func, "arith.subf"), @@ -218,18 +228,24 @@ def bundle_xegpu_fused_attention_schedule( raise PipelineInterrupt() # Fuse the inner attention loop before vectorization. - reduction_tile = layer_params["reduction_tile"] + reduction_tile_vector = layer_params["reduction_tile"] + reduction_tile = reduction_tile_vector[-1] func = transform_to_flash_attention( anytype, func, - reduction_tile, - n_parallel=len(layer_params["wg_tile"]), + reduction_tile_vector, ) # Inline splat-constant generic inputs (e.g. masked-select zero) as # scalars so they are not vectorized into reads of constant tensors. func = transform_ext.fold_scalar_or_splat_constant(func) + lh_transform.cleanup(func) + + # Write the result through an expanded view of the output memref so the GQA + # collapse_shape leaves the write path and bufferization elides the output + # buffer + copy. + func = transform_ext.expand_output_destination(func) if stop_at_stage == "reduction-tiled": raise PipelineInterrupt() @@ -237,18 +253,7 @@ def bundle_xegpu_fused_attention_schedule( # Vectorize func = vectorize(mod, payload_func=func) - # The accumulators of the flash loop are tensors at linalg level, so - # vectorization turns them into a transfer_read/transfer_write pair per - # iteration. Repeat the subset hoisting now that CSE has run, so that all - # accumulators are carried as vector iter_args, i.e. in registers. - reduction_loop = match(func, ops={"scf.for"}) - lh_transform.loop_hoisting(reduction_loop) - enable_fastmath_optimizations(func) - transform.apply_cse(func) - canonicalize(func) - - func = apply_registered_pass(func, "remove-dead-values") lh_transform.cleanup(func) if stop_at_stage == "vectorized": @@ -290,6 +295,19 @@ def bundle_xegpu_fused_attention_schedule( gpu_mod = match(mod, ops={"gpu.module"}) gpu_func = match(gpu_mod, ops={"gpu.func"}) + # Define XeGPU layout parameters + d_head = layer_params["d_head"] + sg_rows = layer_params["sg_rows"] + + # The K/V prefetches cover the same [reduction_tile, d_head] tile as the + # loads, but are distributed over the subgroups. + prefetch_sg_data = layer_params["prefetch_tile"] + prefetch_sg_layout = [ + reduction_tile // prefetch_sg_data[0], + d_head // prefetch_sg_data[1], + ] + prefetch_inst_data = list(prefetch_sg_data) + # Insert prefetches for the K and V tiles of the reduction loop. Each # inserts nb_prefetch prefetches ahead of the loop plus one per iteration, # at induction_var + nb_prefetch * step. This must run before the wg-level @@ -297,17 +315,34 @@ def bundle_xegpu_fused_attention_schedule( # load's descriptor. The layouts of the emitted prefetch_nd ops are set # together with the other wg-level layouts. nb_prefetch = layer_params.get("nb_prefetch", 1) - if nb_prefetch > 0: - reduction_loop = match(gpu_func, ops={"scf.for"}) - kv_load_ops = match_and_split(reduction_loop, ops={"xegpu.load_nd"}, nhandles=2) - for load_op in kv_load_ops: - xegpu.insert_prefetch(load_op, nb_prefetch=nb_prefetch) - transform.apply_cse(gpu_func) - canonicalize(gpu_func) - # Define XeGPU layout parameters - d_head = layer_params["d_head"] - sg_rows = layer_params["sg_rows"] + def _add_prefetch(load_op): + add_prefetch( + load_op, + nb_prefetch, + sg_layout=prefetch_sg_layout, + sg_data=prefetch_sg_data, + inst_data=prefetch_inst_data, + ) + + if nb_prefetch > 0: + # Add prefetch ops to loads within the reduction loop. + reduction_loop = transform_ext.extract_handle( + lh_transform.match_op(gpu_func, "scf.for"), 0, silenceable=True + ) + load_ops = lh_transform.match_op(reduction_loop, "xegpu.load_nd") + with lh_transform.foreach(load_ops) as load_op: + _add_prefetch(load_op) + transform.yield_() + # Handle possible nested scf.for loop + nested_for_ops = lh_transform.match_op(reduction_loop, "scf.for") + with lh_transform.foreach(nested_for_ops) as for_op: + nested_load_ops = lh_transform.match_op(for_op, "xegpu.load_nd") + with lh_transform.foreach(nested_load_ops) as load_op: + _add_prefetch(load_op) + transform.yield_() + transform.yield_() + lh_transform.cleanup(gpu_func) # Q, the attention weights and the output are all [wg_rows, d_head] tiles # that are split by rows over the subgroups. Only the memory ops carry @@ -335,15 +370,6 @@ def bundle_xegpu_fused_attention_schedule( # V operand of the P@V DPAS: [reduction_tile, d_head] v_sg_data = [reduction_tile, d_head] - # The K/V prefetches cover the same [reduction_tile, d_head] tile as the - # loads, but are distributed over the subgroups. - prefetch_sg_data = layer_params["prefetch_tile"] - prefetch_sg_layout = [ - reduction_tile // prefetch_sg_data[0], - d_head // prefetch_sg_data[1], - ] - prefetch_inst_data = list(prefetch_sg_data) - # Set layout attributes for xegpu.store_nd ops. store_nd_op = match_and_split(gpu_func, ops={"xegpu.store_nd"}, nhandles=1)[0] xegpu.set_anchor_layout( From b7f77a051b8d10f0519964f9e12320216646132f Mon Sep 17 00:00:00 2001 From: Tuomas Karna Date: Fri, 16 Oct 2026 23:25:37 +0300 Subject: [PATCH 12/18] kernel_bench: dispatch correct params for MHA and GQA case --- examples/xegpu/kernel_bench.py | 59 +++++++++++++++++++++++++--------- 1 file changed, 44 insertions(+), 15 deletions(-) diff --git a/examples/xegpu/kernel_bench.py b/examples/xegpu/kernel_bench.py index 49c0c9ba..a8a37fe6 100644 --- a/examples/xegpu/kernel_bench.py +++ b/examples/xegpu/kernel_bench.py @@ -275,21 +275,49 @@ def infer_parameters( write_bytes = int(batch_size * n_head * n_ctx * d_head * nbytes) assert d_head == 64, f"d_head must be 64, got {d_head}" - layer_params = { - "layer_kind": "attention", - "batch_size": batch_size, - "n_head": n_head, - "n_ctx": n_ctx, - "d_head": d_head, - "wg_tile": [1, 1, 128], - "sg_rows": 16, - "subgroup_size": 16, - "reduction_tile": 64, - "q_load_tile": [16, 16], - "v_load_tile": [32, 32], - "prefetch_tile": [16, 32], - "nb_prefetch": 1, - } + + max_elemwise_dims = 0 + for layer in [s for s in func_metadata["layers"] if s["kind"] == "elemwise"]: + max_elemwise_dims = max(max_elemwise_dims, len(layer["shape"])) + + if max_elemwise_dims == 4: + # MHA case + layer_params = { + "layer_kind": "attention", + "batch_size": batch_size, + "n_head": n_head, + "n_ctx": n_ctx, + "d_head": d_head, + "wg_tile": [1, 1, 128], + "sg_rows": 16, + "subgroup_size": 16, + "reduction_tile": [0, 0, 0, 64], + "q_load_tile": [16, 16], + "v_load_tile": [32, 32], + "prefetch_tile": [16, 32], + "nb_prefetch": 1, + } + elif max_elemwise_dims == 5: + # GQA case + layer_params = { + "layer_kind": "attention", + "batch_size": batch_size, + "n_head": n_head, + "n_ctx": n_ctx, + "d_head": d_head, + "wg_tile": [1, 1, 1, 128], + "sg_rows": 16, + "subgroup_size": 16, + "reduction_tile": [0, 0, 0, 0, 64], + "q_load_tile": [16, 16], + "v_load_tile": [32, 32], + "prefetch_tile": [16, 32], + "nb_prefetch": 1, + } + else: + raise ValueError( + f"Unsupported attention layer dimensionality: {max_elemwise_dims}" + ) schedule_params = ScheduleParameters([layer_params]) schedule_kind = "attention" @@ -947,6 +975,7 @@ def parser_cli_args(): "initial", "cleanup", "tiled", + "reduction-tiled", "vectorized", "bufferized", "gpu-outlining", From 2cfad213248921e6ab74006a5d21560f6e829864 Mon Sep 17 00:00:00 2001 From: Tuomas Karna Date: Fri, 16 Oct 2026 23:44:14 +0300 Subject: [PATCH 13/18] update fused_attention example --- examples/xegpu/fused_attention.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/xegpu/fused_attention.py b/examples/xegpu/fused_attention.py index fab111e0..49e7dfe8 100644 --- a/examples/xegpu/fused_attention.py +++ b/examples/xegpu/fused_attention.py @@ -348,7 +348,7 @@ def parse_cli(): "wg_tile": [1, args.wg_rows], "sg_rows": args.sg_rows, "subgroup_size": args.subgroup_size, - "reduction_tile": args.reduction_tile, + "reduction_tile": [0, 0, args.reduction_tile], "q_load_tile": args.q_load_tile, "v_load_tile": args.v_load_tile, "prefetch_tile": args.prefetch_tile, From d1c05ea553d2ac39632d9edf7e8601ce715cfdcb Mon Sep 17 00:00:00 2001 From: Tuomas Karna Date: Sat, 17 Oct 2026 00:04:13 +0300 Subject: [PATCH 14/18] xegpu examples: update lit config --- examples/xegpu/lit.local.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/xegpu/lit.local.cfg b/examples/xegpu/lit.local.cfg index 17c1661c..6a32c0bc 100644 --- a/examples/xegpu/lit.local.cfg +++ b/examples/xegpu/lit.local.cfg @@ -4,5 +4,6 @@ config.excludes = [ "nanoGPT_schedule.py", "llama3_payload.py", "llama3_schedule.py", + "llama-bench_1B_attention.py", "tune_utils.py", ] From 8fb68ae50cff1e44d476d4572c9762483aa95655 Mon Sep 17 00:00:00 2001 From: Tuomas Karna Date: Sat, 17 Oct 2026 00:08:57 +0300 Subject: [PATCH 15/18] fix rewrite_attention_row_mask test --- test/transform/test_rewrite_attention_row_mask.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test/transform/test_rewrite_attention_row_mask.py b/test/transform/test_rewrite_attention_row_mask.py index 4a36a93e..704160d0 100644 --- a/test/transform/test_rewrite_attention_row_mask.py +++ b/test/transform/test_rewrite_attention_row_mask.py @@ -654,9 +654,7 @@ def test_score_producer_fusion(): module = ir.Module.parse(source) with schedule_boilerplate() as (schedule, sequence): func = lh_transform.match_op(sequence.bodyTarget, "func.func") - transform_to_flash_attention( - transform.AnyOpType.get(), func, 2, n_parallel=1 - ) + transform_to_flash_attention(transform.AnyOpType.get(), func, [0, 2]) transform.yield_([]) schedule.body.operations[0].apply(module.operation) assert module.operation.verify() From 240bf062438245d9509ef1ab782f2d5c21eddfa3 Mon Sep 17 00:00:00 2001 From: Tuomas Karna Date: Mon, 19 Oct 2026 20:56:54 +0300 Subject: [PATCH 16/18] gpu pipeline: add fastmath flags at vector level --- lighthouse/schedule/xegpu/vectorize_schedule.py | 4 ++++ lighthouse/schedule/xegpu/wg_tiling_schedule.py | 7 +------ 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/lighthouse/schedule/xegpu/vectorize_schedule.py b/lighthouse/schedule/xegpu/vectorize_schedule.py index 6d197b38..869906a6 100644 --- a/lighthouse/schedule/xegpu/vectorize_schedule.py +++ b/lighthouse/schedule/xegpu/vectorize_schedule.py @@ -2,6 +2,8 @@ from mlir.dialects import transform from lighthouse.schedule import schedule_boilerplate +from .attention_fusion import enable_fastmath_optimizations + from . import lowering_common @@ -29,6 +31,8 @@ def vectorize_schedule( deduplicate=True, ) func = lowering_common.vectorize(payload_mod, payload_func=func) + enable_fastmath_optimizations(func) + transform.yield_() return schedule diff --git a/lighthouse/schedule/xegpu/wg_tiling_schedule.py b/lighthouse/schedule/xegpu/wg_tiling_schedule.py index 624913db..e74d3d63 100644 --- a/lighthouse/schedule/xegpu/wg_tiling_schedule.py +++ b/lighthouse/schedule/xegpu/wg_tiling_schedule.py @@ -10,10 +10,7 @@ ) from lighthouse.schedule import schedule_boilerplate from lighthouse.dialects.transform import transform_ext -from .attention_fusion import ( - transform_to_flash_attention, - enable_fastmath_optimizations, -) +from .attention_fusion import transform_to_flash_attention from .lowering_common import get_payload_func @@ -238,8 +235,6 @@ def apply_attention_tiling(func: ir.Operation) -> ir.Operation: # Inline splat-constant generic inputs (e.g. masked-select zero) as # scalars so they are not vectorized into reads of constant tensors. func = transform_ext.fold_scalar_or_splat_constant(func) - - enable_fastmath_optimizations(func) lh_transform.cleanup(func) # Write the result through an expanded view of the output memref so the GQA From a85686e3119649af3f01efc7a5ce3b9efcbe86f8 Mon Sep 17 00:00:00 2001 From: Tuomas Karna Date: Mon, 19 Oct 2026 21:28:51 +0300 Subject: [PATCH 17/18] fused_attention_schedule: fix prefetch insertion loop --- .../xegpu/fused_attention_schedule.py | 47 ++++++------------- 1 file changed, 15 insertions(+), 32 deletions(-) diff --git a/lighthouse/schedule/xegpu/fused_attention_schedule.py b/lighthouse/schedule/xegpu/fused_attention_schedule.py index ba371a66..24b4231e 100644 --- a/lighthouse/schedule/xegpu/fused_attention_schedule.py +++ b/lighthouse/schedule/xegpu/fused_attention_schedule.py @@ -316,30 +316,23 @@ def bundle_xegpu_fused_attention_schedule( # together with the other wg-level layouts. nb_prefetch = layer_params.get("nb_prefetch", 1) - def _add_prefetch(load_op): - add_prefetch( - load_op, - nb_prefetch, - sg_layout=prefetch_sg_layout, - sg_data=prefetch_sg_data, - inst_data=prefetch_inst_data, - ) - if nb_prefetch > 0: # Add prefetch ops to loads within the reduction loop. - reduction_loop = transform_ext.extract_handle( - lh_transform.match_op(gpu_func, "scf.for"), 0, silenceable=True - ) - load_ops = lh_transform.match_op(reduction_loop, "xegpu.load_nd") - with lh_transform.foreach(load_ops) as load_op: - _add_prefetch(load_op) - transform.yield_() - # Handle possible nested scf.for loop - nested_for_ops = lh_transform.match_op(reduction_loop, "scf.for") - with lh_transform.foreach(nested_for_ops) as for_op: - nested_load_ops = lh_transform.match_op(for_op, "xegpu.load_nd") - with lh_transform.foreach(nested_load_ops) as load_op: - _add_prefetch(load_op) + all_load_ops = lh_transform.match_op(gpu_func, "xegpu.load_nd") + with lh_transform.foreach(all_load_ops) as load_op: + # Loop over each load op. + parent_for = transform.get_parent_op( + anytype, load_op, op_name="scf.for", allow_empty_results=True + ) + with lh_transform.foreach(parent_for): + # Add prefetch if parent is an scf.for loop. + add_prefetch( + load_op, + nb_prefetch, + sg_layout=prefetch_sg_layout, + sg_data=prefetch_sg_data, + inst_data=prefetch_inst_data, + ) transform.yield_() transform.yield_() lh_transform.cleanup(gpu_func) @@ -405,16 +398,6 @@ def _add_prefetch(load_op): inst_data=v_load_inst_data, ) - # Set layout for all K/V xegpu.prefetch_nd ops - if nb_prefetch > 0: - prefetch_ops = match(gpu_func, ops={"xegpu.prefetch_nd"}) - xegpu.set_anchor_layout( - prefetch_ops, - sg_layout=prefetch_sg_layout, - sg_data=prefetch_sg_data, - inst_data=prefetch_inst_data, - ) - # Set layout for xegpu.dpas ops (2 total: Q@K^T and P@V) dpas_ops = match_and_split(gpu_func, ops={"xegpu.dpas"}, nhandles=2) From 46ca0eeb27d7b2f0c74eb63590da14ca3a6989db Mon Sep 17 00:00:00 2001 From: Tuomas Karna Date: Mon, 19 Oct 2026 21:29:35 +0300 Subject: [PATCH 18/18] annotate_layouts_schedule: fix prefetch insertion loop --- .../xegpu/annotate_layouts_schedule.py | 39 ++++++++----------- 1 file changed, 17 insertions(+), 22 deletions(-) diff --git a/lighthouse/schedule/xegpu/annotate_layouts_schedule.py b/lighthouse/schedule/xegpu/annotate_layouts_schedule.py index 1e8f1417..29228054 100644 --- a/lighthouse/schedule/xegpu/annotate_layouts_schedule.py +++ b/lighthouse/schedule/xegpu/annotate_layouts_schedule.py @@ -215,6 +215,8 @@ def annotate_attention(gpu_func: ir.Operation, anchor_op: ir.Operation): # ) # TODO infer the layout parameters - hardcoded for now + anytype = transform.AnyOpType.get() + # Insert prefetches for the K and V tiles of the reduction loop. Each # inserts nb_prefetch prefetches ahead of the loop plus one per iteration, # at induction_var + nb_prefetch * step. This must run before the wg-level @@ -226,30 +228,23 @@ def annotate_attention(gpu_func: ir.Operation, anchor_op: ir.Operation): prefetch_sg_layout = [4, 2] prefetch_inst_data = list(prefetch_sg_data) - def _add_prefetch(load_op): - add_prefetch( - load_op, - nb_prefetch, - sg_layout=prefetch_sg_layout, - sg_data=prefetch_sg_data, - inst_data=prefetch_inst_data, - ) - if nb_prefetch > 0: # Add prefetch ops to loads within the reduction loop. - reduction_loop = transform_ext.extract_handle( - lh_transform.match_op(gpu_func, "scf.for"), 0, silenceable=True - ) - load_ops = lh_transform.match_op(reduction_loop, "xegpu.load_nd") - with lh_transform.foreach(load_ops) as load_op: - _add_prefetch(load_op) - transform.yield_() - # Handle possible nested scf.for loop - nested_for_ops = lh_transform.match_op(reduction_loop, "scf.for") - with lh_transform.foreach(nested_for_ops) as for_op: - nested_load_ops = lh_transform.match_op(for_op, "xegpu.load_nd") - with lh_transform.foreach(nested_load_ops) as load_op: - _add_prefetch(load_op) + all_load_ops = lh_transform.match_op(gpu_func, "xegpu.load_nd") + with lh_transform.foreach(all_load_ops) as load_op: + # Loop over each load op. + parent_for = transform.get_parent_op( + anytype, load_op, op_name="scf.for", allow_empty_results=True + ) + with lh_transform.foreach(parent_for): + # Add prefetch if parent is an scf.for loop. + add_prefetch( + load_op, + nb_prefetch, + sg_layout=prefetch_sg_layout, + sg_data=prefetch_sg_data, + inst_data=prefetch_inst_data, + ) transform.yield_() transform.yield_() lh_transform.cleanup(gpu_func)