Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion examples/xegpu/fused_attention.py
Original file line number Diff line number Diff line change
Expand Up @@ -348,7 +348,7 @@ def parse_cli():
"wg_tile": [1, args.wg_rows],
"sg_rows": args.sg_rows,
"subgroup_size": args.subgroup_size,
"reduction_tile": args.reduction_tile,
"reduction_tile": [0, 0, args.reduction_tile],
"q_load_tile": args.q_load_tile,
"v_load_tile": args.v_load_tile,
"prefetch_tile": args.prefetch_tile,
Expand Down
81 changes: 63 additions & 18 deletions examples/xegpu/kernel_bench.py
Original file line number Diff line number Diff line change
Expand Up @@ -275,21 +275,49 @@ def infer_parameters(
write_bytes = int(batch_size * n_head * n_ctx * d_head * nbytes)

assert d_head == 64, f"d_head must be 64, got {d_head}"
layer_params = {
"layer_kind": "attention",
"batch_size": batch_size,
"n_head": n_head,
"n_ctx": n_ctx,
"d_head": d_head,
"wg_tile": [1, 1, 128],
"sg_rows": 16,
"subgroup_size": 16,
"reduction_tile": 64,
"q_load_tile": [16, 16],
"v_load_tile": [32, 32],
"prefetch_tile": [16, 32],
"nb_prefetch": 1,
}

max_elemwise_dims = 0
for layer in [s for s in func_metadata["layers"] if s["kind"] == "elemwise"]:
max_elemwise_dims = max(max_elemwise_dims, len(layer["shape"]))

if max_elemwise_dims == 4:
# MHA case
layer_params = {
"layer_kind": "attention",
"batch_size": batch_size,
"n_head": n_head,
"n_ctx": n_ctx,
"d_head": d_head,
"wg_tile": [1, 1, 128],
"sg_rows": 16,
"subgroup_size": 16,
"reduction_tile": [0, 0, 0, 64],
"q_load_tile": [16, 16],
"v_load_tile": [32, 32],
"prefetch_tile": [16, 32],
"nb_prefetch": 1,
}
elif max_elemwise_dims == 5:
# GQA case
layer_params = {
"layer_kind": "attention",
"batch_size": batch_size,
"n_head": n_head,
"n_ctx": n_ctx,
"d_head": d_head,
"wg_tile": [1, 1, 1, 128],
"sg_rows": 16,
"subgroup_size": 16,
"reduction_tile": [0, 0, 0, 0, 64],
"q_load_tile": [16, 16],
"v_load_tile": [32, 32],
"prefetch_tile": [16, 32],
"nb_prefetch": 1,
}
else:
raise ValueError(
f"Unsupported attention layer dimensionality: {max_elemwise_dims}"
)

schedule_params = ScheduleParameters([layer_params])
schedule_kind = "attention"
Expand Down Expand Up @@ -918,6 +946,12 @@ def parser_cli_args():
nargs="+",
help="IDs of specific benchmarks to execute (default: all)",
)
parser.add_argument(
"-i",
"--input",
nargs="+",
help="Input Python kernel files. Can be used instead of --level and --benchmark to specify custom kernels.",
)
parser.add_argument(
"--datatype",
type=str,
Expand All @@ -941,6 +975,7 @@ def parser_cli_args():
"initial",
"cleanup",
"tiled",
"reduction-tiled",
"vectorized",
"bufferized",
"gpu-outlining",
Expand Down Expand Up @@ -995,11 +1030,21 @@ def parser_cli_args():
args = parser_cli_args()
kb_level = args.level
benchmarks = args.benchmark
input_files = args.input
if input_files is not None and kb_level is not None and benchmarks is not None:
raise ValueError("Cannot set both --input, and --level and --benchmark. ")
if input_files is None and kb_level is None and benchmarks is None:
raise ValueError("Either --input or --level and --benchmark must be specified.")

stop_at_stage = args.dump_kernel
input_shapes = parse_input_shapes(args.input_shapes)

kb_pattern = f"level{kb_level}/*.py"
bench_list = get_benchmarks(kb_pattern, include=benchmarks)
if input_files is None:
kb_pattern = f"level{kb_level}/*.py"
bench_list = get_benchmarks(kb_pattern, include=benchmarks)
else:
bench_list = [(i + 1, Path(f)) for i, f in enumerate(input_files)]
kb_level = 0

if args.dump_csv and not stop_at_stage:
csv_file = "out_kernelbench.csv"
Expand All @@ -1008,7 +1053,7 @@ def parser_cli_args():
csv_logger = None

for bench_id, bench_path in bench_list:
short_path = bench_path.parent.name + "/" + bench_path.name
short_path = Path(*bench_path.parts[-2:])
print("-" * 80)
print(f"Executing benchmark: {short_path}", flush=True)
entry = {
Expand Down
1 change: 1 addition & 0 deletions examples/xegpu/lit.local.cfg
Original file line number Diff line number Diff line change
Expand Up @@ -4,5 +4,6 @@ config.excludes = [
"nanoGPT_schedule.py",
"llama3_payload.py",
"llama3_schedule.py",
"llama-bench_1B_attention.py",
"tune_utils.py",
]
54 changes: 54 additions & 0 deletions examples/xegpu/llama-bench_1B_attention.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
import torch
import torch.nn as nn


class Model(nn.Module):
"""
Llama3.2-1B scaled dot-product attention (GQA + causal), the realistic
counterpart to KernelBench level1-97.

Differences from the synthetic level1-97 SDPA:
* d_head = 64 (Llama3.2-1B), the per-head reduction dimension, within the
current <= 64 lowering limit. (Llama3-8B uses d_head = 128, which still
needs the new Intel-GPU head tiling.)

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think you could also reduce the reduction tile size to preserve registers. that way you may not need head tiling.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good point, for d_head=128 that might be sufficient.

* Grouped-query attention: 32 query heads share 8 key/value heads
(num_heads / num_kv_heads = 4). K/V carry fewer heads than Q and are
broadcast across the group (enable_gqa=True).
* Causal masking (is_causal=True), as in autoregressive decoding/prefill.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

here how is the causal masking handled? is masking applied to QK^t?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, causal masking adds a strict upper diagonal mask with -inf values (the rest being zeros) to the QK^T results. The subsequent -inf handling ensures that these values do not contribute to the attention.


RoPE is intentionally NOT applied here; the rotary embedding is a separate
stepping-stone (llama-bench_4) because rotate_half is a slice+concat shuffle
on the innermost (head) dimension rather than a plain matmul/elementwise op.
Q/K/V are taken as already-projected inputs, matching level1-97.
"""

def __init__(self):
super().__init__()

def forward(
self, Q: torch.Tensor, K: torch.Tensor, V: torch.Tensor
) -> torch.Tensor:
# Q: (batch, num_heads, n_ctx, d_head); K/V: (batch, num_kv_heads, n_ctx, d_head)
causal = False # NOTE handle causal masking in the lowering
return torch.nn.functional.scaled_dot_product_attention(
Q, K, V, is_causal=causal, enable_gqa=True
)


# Llama3.2-1B attention config.
batch_size = 4
n_ctx = 512
num_heads = 32 # query heads
num_kv_heads = 8 # grouped key/value heads (GQA group size = 4)
d_head = 64 # per-head reduction dim; within the current <= 64 cap


def get_inputs():
Q = torch.rand(batch_size, num_heads, n_ctx, d_head)
K = torch.rand(batch_size, num_kv_heads, n_ctx, d_head)
V = torch.rand(batch_size, num_kv_heads, n_ctx, d_head)
return [Q, K, V]


def get_init_inputs():
return []
2 changes: 2 additions & 0 deletions lighthouse/dialects/transform/transform_ext/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
from .ops.convert_func_results_to_args import convert_func_results_to_args
from .ops.narrow_vector_contract_operands import narrow_vector_contract_operands
from .ops.emit_definite_failure import emit_definite_failure
from .ops.expand_output_destination import expand_output_destination
from .ops.extract_handle import extract_handle
from .ops.get_tileable_consumers import get_tileable_consumers
from .ops.get_tiling_sizes import get_tiling_sizes
Expand Down Expand Up @@ -54,6 +55,7 @@
"compute_sg_layout",
"convert_func_results_to_args",
"emit_definite_failure",
"expand_output_destination",
"extract_handle",
"filter_by_name",
"filter_contraction_ops",
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
from mlir import ir
from mlir.dialects import ext, transform, memref, bufferization
from mlir.dialects.transform import DiagnosedSilenceableFailure

from lighthouse.dialects.transform.transform_ext import TransformExtensionDialect


class ExpandOutputDestinationOp(
TransformExtensionDialect.Operation, name="expand_output_destination"
):
"""Remove a reshape from a result's write path so it bufferizes in place.

Rewrites ``materialize_in_destination(collapse_shape(X), D)`` into
``materialize_in_destination(X, memref.expand_shape(D))``.

WG-tiled attention writes its result in the GQA-expanded N-D layout, but the
output memref is the collapsed (N-1)-D layout, so a ``tensor.collapse_shape``

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I wonder if there is some existing transform op to handle this. I remember running into similar issues.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not that I know of. At the moment I can't really think of a generic way for fixing this upstream other than a specialized bufferization.materialize_in_destination transform like the one added here. The IR pattern essentially is:

func.func @main(..., %arg3: memref<4x32x512x64xbf16>) {
  %4 = tensor.empty() : tensor<4x8x4x512x64xbf16>
  %5 = scf.forall ... shared_outs(%arg8 = %4) -> (tensor<4x8x4x512x64xbf16>) { ... }
  %collapsed = tensor.collapse_shape %5 [[0], [1, 2], [3], [4]] : tensor<4x8x4x512x64xbf16> into
      tensor<4x32x512x64xbf16>
  bufferization.materialize_in_destination %collapsed in restrict writable %arg3 :
      (tensor<4x32x512x64xbf16>, memref<4x32x512x64xbf16>) -> ()

sits between the loop result and the destination. That reshape makes the loop
result and the destination differ in shape, so empty-tensor elimination cannot
fold the loop's output into the destination and bufferization materializes a
separate buffer and copies it out. Reshaping the destination up to the loop's
layout removes the reshape from the write path, so elimination fires and the
loop writes the output in place.

Only rewrites ``materialize_in_destination`` ops whose destination is a memref
and whose source is a single ``tensor.collapse_shape``; a no-op otherwise.
"""

target: ext.Operand[transform.AnyOpType]
result: ext.Result[transform.AnyOpType[()]] = ext.infer_result()

@classmethod
def attach_interface_impls(cls, ctx=None):
cls.TransformOpInterfaceModel.attach(cls.OPERATION_NAME, context=ctx)
cls.MemoryEffectsOpInterfaceModel.attach(cls.OPERATION_NAME, context=ctx)

class TransformOpInterfaceModel(transform.TransformOpInterface):
@staticmethod
def apply(op, _rewriter, results, state) -> DiagnosedSilenceableFailure:
targets = list(state.get_payload_ops(op.target))
for func in targets:
_rewrite(func)
results.set_ops(op.result, targets)
return DiagnosedSilenceableFailure.Success

@staticmethod
def allow_repeated_handle_operands(_op) -> bool:
return False

class MemoryEffectsOpInterfaceModel(ir.MemoryEffectsOpInterface):
@staticmethod
def get_effects(op):
return (
transform.only_reads_handle(op.op_operands)
+ transform.produces_handle(op.results)
+ transform.modifies_payload()
)


def _reassociation(collapse: ir.OpView) -> list[list[int]]:
return [
[ir.IntegerAttr(idx).value for idx in ir.ArrayAttr(group)]
for group in ir.ArrayAttr(collapse.attributes["reassociation"])
]


def _rewrite(func: ir.OpView) -> None:
materializes = [
o
for o in func.regions[0].blocks[0].operations
if o.operation.name == "bufferization.materialize_in_destination"
]
for mat in materializes:
source, dest = mat.operands[0], mat.operands[1]
if not isinstance(dest.type, ir.MemRefType):
continue
collapse = source.owner
if collapse.operation.name != "tensor.collapse_shape":
continue
expanded = collapse.operands[0]
expanded_type = ir.RankedTensorType(expanded.type)
with ir.InsertionPoint(mat):
dest_type = ir.MemRefType.get(
expanded_type.shape, expanded_type.element_type
)
dest_expanded = memref.expand_shape(
dest_type,
dest,
_reassociation(collapse),
[],
static_output_shape=list(expanded_type.shape),
)
bufferization.materialize_in_destination(
None, expanded, dest_expanded, restrict=True, writable=True
)
mat.operation.erase()


def expand_output_destination(
target: ir.Value[transform.AnyOpType],
) -> ir.Value[transform.AnyOpType]:
"""Rewrite result ``materialize_in_destination`` to write in place."""
return ExpandOutputDestinationOp(target=target).result
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
from mlir import ir
from mlir.dialects import ext, transform
from mlir.dialects import ext, transform, linalg, vector
from mlir.dialects.transform import DiagnosedSilenceableFailure

from lighthouse.dialects.transform.transform_ext import TransformExtensionDialect
Expand All @@ -17,10 +17,7 @@ class InferXeGPUAttentionParamsOp(
directly to the tiling routines without splitting; `reduction_tile` is a
scalar i64 param.

NOTE: placeholder implementation; the tiles are hard-coded for now and the
analysis deriving them from `target` will be added in a follow-up. Later
schedules will additionally need sg_rows, q/v load tiles and prefetch
parameters; those results will be added here when required.
NOTE: This is just a placeholder implementation with hard-coded tile sizes.

Args:
target: Handle to the attention anchor op(s).
Expand Down Expand Up @@ -51,13 +48,50 @@ def apply(
results: transform.TransformResults,
state: transform.TransformState,
) -> DiagnosedSilenceableFailure:
# TODO: derive the tiles from `target`; placeholders for now.
i64 = ir.IntegerType.get_signless(64)
target_ops = state.get_payload_ops(op.target)
if len(target_ops) != 1:
# expecting a single anchor op
return DiagnosedSilenceableFailure.SilenceableFailure
target_op = target_ops[0]
rank = ir.ShapedType(target_op.results[0].type).rank

size_attrs = InferXeGPUAttentionParamsOp._size_attrs
results.set_params(op.wg_tile, size_attrs([1, 1, 128]))
results.set_params(op.sg_tile, size_attrs([0, 0, 16]))
# reduction_tile is a scalar; generalize to an array later if needed.
results.set_params(op.reduction_tile, [ir.IntegerAttr.get(i64, 64)])

wg_rows, sg_rows, reduction_tile = 128, 16, 64

if isinstance(target_op, linalg.GenericOp):
# Assume non-tiled linalg.matmul
# WG/SG row sizes and the reduction (K/V seq) tile are hard-coded.
# Tile every leading parallel dim (incl. the GQA group) by 1, the
# query-row dim by the WG row size, and leave d_head untiled.
if rank == 4: # plain MHA: (batch, head, query_row, d_head)
wg, sg = [1, 1, wg_rows], [0, 0, sg_rows]
red = [0, 0, 0, reduction_tile]
elif rank == 5: # GQA: (batch, kv_head, group, query_row, d_head)
wg, sg = [1, 1, 1, wg_rows], [0, 0, 0, sg_rows]
red = [0, 0, 0, 0, reduction_tile]
else:
op.location.emit_error(
"infer_xegpu_attention_params: unsupported attention leaf "
f"rank {rank}; expected 4 (MHA) or 5 (GQA)"
)
return DiagnosedSilenceableFailure.SilenceableFailure
elif isinstance(target_op, vector.ContractionOp) and rank == 2:
# Assume tiled vector.contract, op's result shape is
# (wg_rows, reduction_tile)
wg_rows, _ = ir.ShapedType(target_op.results[0].type).shape
wg = [wg_rows, 0]
sg = [sg_rows, 0]
red = [0, reduction_tile]
else:
op.location.emit_error(
"infer_xegpu_attention_params: unsupported attention leaf "
f"op {target_op.operation.name} with rank {rank}"
)
return DiagnosedSilenceableFailure.SilenceableFailure
results.set_params(op.wg_tile, size_attrs(wg))
results.set_params(op.sg_tile, size_attrs(sg))
results.set_params(op.reduction_tile, size_attrs(red))
return DiagnosedSilenceableFailure.Success

@staticmethod
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -238,9 +238,11 @@ def _trace_producer_scale(contraction, operand_index):
body = op.regions[0].blocks[0]
args = list(body.arguments)[: len(inputs)]
scalar = _producer(body.operations[-1].operands[0])
if scalar is None:
return None, None
# Only constants may accompany the selected scalar operation.
if any(
inner.operation != getattr(scalar, "operation", None)
inner.operation != scalar.operation
and not isinstance(inner, (arith.ConstantOp, linalg.YieldOp))
for inner in body.operations
):
Expand Down
Loading
Loading