Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
76 changes: 55 additions & 21 deletions examples/xegpu/fused_attention.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
# RUN: %PYTHON %s --dump-kernel=xegpu-wg | FileCheck %s
# RUN: %PYTHON %s --dump-kernel=xegpu-wg --wg-rows 64 --sg-rows 32 --reduction-tile 32 --q-load-tile 16 16 --prefetch-tile 32 16 --nb-prefetch 2 | FileCheck %s
# CHECK: module attributes {gpu.container_module} {

"""
Expand All @@ -20,8 +21,56 @@
from lighthouse.ingress.mlir_gen.gpu_attention_payload import (
generate_gpu_attention_payload,
)
from lighthouse.schedule.xegpu import fused_attention_schedule, xegpu_to_binary
from lighthouse.schedule.parameters import ScheduleParameters
from lighthouse.schedule import xegpu


def get_xegpu_pipeline(parameters: dict, stop_at_stage: str) -> list:
wg_tile = parameters.get("wg_tile", None)
sg_tile = parameters.get("sg_tile", None)
reduction_tile = parameters.get("reduction_tile", None)
nb_prefetch = parameters.get("nb_prefetch", None)
prefetch_tile = parameters.get("prefetch_tile", None)
q_load_tile = parameters.get("q_load_tile", None)
v_load_tile = parameters.get("v_load_tile", None)
schedules = []
if stop_at_stage == "initial":
return schedules
schedules.append(xegpu.cleanup_schedule())
if stop_at_stage == "cleanup":
return schedules
schedules.append(
xegpu.wg_tiling_schedule(
wg_tile=wg_tile,
reduction_tile=reduction_tile,
)
)
if stop_at_stage == "tiled":
return schedules
schedules.append(xegpu.vectorize_schedule())
if stop_at_stage == "vectorized":
return schedules
schedules.append(xegpu.bufferize_schedule())
if stop_at_stage == "bufferized":
return schedules
schedules.append(xegpu.outline_gpu_func_schedule(sg_tile=sg_tile))
if stop_at_stage == "gpu-outlining":
return schedules
schedules.append(xegpu.vector_to_xegpu_schedule())
if stop_at_stage == "xegpu-initial":
return schedules
schedules.append(
xegpu.annotate_layouts_schedule(
sg_tile=sg_tile,
nb_prefetch=nb_prefetch,
prefetch_tile=prefetch_tile,
q_load_tile=q_load_tile,
v_load_tile=v_load_tile,
)
)
if stop_at_stage == "xegpu-wg":
return schedules
return schedules


def fused_attention_complexity(
Expand Down Expand Up @@ -187,18 +236,14 @@ def schedule_modules(
"""Generate transform schedule for fused attention."""
schedules = []
schedules.append(Runner.get_bench_wrapper_schedule(self.payload_function_name))

schedules.append(
fused_attention_schedule(
stop_at_stage=stop_at_stage,
params=parameters,
)
schedules.extend(
get_xegpu_pipeline(parameters=parameters[0], stop_at_stage=stop_at_stage)
)

if stop_at_stage and stop_at_stage != "final":
return schedules

schedules.append(xegpu_to_binary())
schedules.append(xegpu.xegpu_to_binary())

return schedules

Expand Down Expand Up @@ -247,12 +292,6 @@ def parse_cli():
default=16,
help="Number of Q*K^T*V rows computed by each subgroup",
)
parser.add_argument(
"--subgroup-size",
type=int,
default=16,
help="Subgroup size",
)
parser.add_argument(
"--reduction-tile",
type=int,
Expand Down Expand Up @@ -308,8 +347,8 @@ def parse_cli():
type=str,
choices=[
"initial",
"cleanup",
"tiled",
"reduction-tiled",
"vectorized",
"bufferized",
"gpu-outlining",
Expand Down Expand Up @@ -341,13 +380,8 @@ def parse_cli():

layer_params = {
"layer_kind": "attention",
"batch_size": args.batch_size,
"n_head": args.n_head,
"n_ctx": args.n_ctx,
"d_head": args.d_head,
"wg_tile": [1, args.wg_rows],
"sg_rows": args.sg_rows,
"subgroup_size": args.subgroup_size,

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Note that I'm dropping the subgroup_size option for now. It is essentially hard-coded to 16 in the attention schedule. We can turn it into a parameter when needed.

"sg_tile": [0, args.sg_rows],
"reduction_tile": [0, 0, args.reduction_tile],
"q_load_tile": args.q_load_tile,
"v_load_tile": args.v_load_tile,
Expand Down
23 changes: 5 additions & 18 deletions examples/xegpu/kernel_bench.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
# RUN: %PYTHON %s -l 2 -b 9 --dump-kernel=xegpu-wg | FileCheck %s
# RUN: %PYTHON %s -l 1 -b 97 --input-shapes 32x32x512x64 32x32x512x64 32x32x512x64 --dump-kernel xegpu-wg | FileCheck %s
# RUN: %PYTHON %s -l 2 -b 9 --dump-kernel=xegpu-wg --use-gpu-pipeline | FileCheck %s
# RUN: %PYTHON %s -l 1 -b 97 --input-shapes 32x32x512x64 32x32x512x64 32x32x512x64 --dump-kernel xegpu-wg --use-gpu-pipeline | FileCheck %s
# REQUIRES: torch
# REQUIRES: kernel_bench

Expand Down Expand Up @@ -70,7 +71,6 @@
elemwise_schedule,
xegpu_to_binary,
reduction_schedule,
fused_attention_schedule,
)
from lighthouse.pipeline.helper import PipelineInterrupt
from lighthouse.ingress.torch import gpu_backend, TargetDialect
Expand Down Expand Up @@ -284,13 +284,8 @@ def infer_parameters(
# MHA case
layer_params = {
"layer_kind": "attention",
"batch_size": batch_size,
"n_head": n_head,
"n_ctx": n_ctx,
"d_head": d_head,
"wg_tile": [1, 1, 128],
"sg_rows": 16,
"subgroup_size": 16,
"sg_tile": [0, 0, 16],
"reduction_tile": [0, 0, 0, 64],
"q_load_tile": [16, 16],
"v_load_tile": [32, 32],
Expand All @@ -301,13 +296,8 @@ def infer_parameters(
# GQA case
layer_params = {
"layer_kind": "attention",
"batch_size": batch_size,
"n_head": n_head,
"n_ctx": n_ctx,
"d_head": d_head,
"wg_tile": [1, 1, 1, 128],
"sg_rows": 16,
"subgroup_size": 16,
"sg_tile": [0, 0, 16],
"reduction_tile": [0, 0, 0, 0, 64],
"q_load_tile": [16, 16],
"v_load_tile": [32, 32],
Expand Down Expand Up @@ -618,10 +608,7 @@ def lower_to_llvm(
stop_at_stage=stop_at_stage,
)
elif schedule_kind == "attention":
schedule = fused_attention_schedule(
params=schedule_params,
stop_at_stage=stop_at_stage,
)
raise NotImplementedError("Attention kernels require --use-gpu-pipeline.")
else:
raise ValueError(f"Unsupported schedule kind: {schedule_kind}")
pipeline = [schedule]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,10 @@ def flatten(handles):
layout = r // s
if w > 0 and s > 0:
layout = w // s
if layout < 1:
raise ValueError(
f"Invalid sg_layout value: {layout}. wg={w}, sg={s}, red={r}."
)
sg_layout.append(layout)
# sg_data tracks the tiled dim: subgroup tile, else reduction.
sg_data.append(s if s > 0 else (r if r > 0 else 1))
Expand Down
40 changes: 28 additions & 12 deletions lighthouse/dialects/transform/transform_ext/ops/extract_handle.py
Original file line number Diff line number Diff line change
@@ -1,29 +1,35 @@
from typing import TypeVar

from mlir import ir
from mlir.dialects import ext, transform
from mlir.dialects._ods_common import get_op_result_or_value
from mlir.dialects.transform import DiagnosedSilenceableFailure

from lighthouse.dialects.transform.transform_ext import TransformExtensionDialect


HandleType = TypeVar("HandleType", bound=transform.AnyOpType | transform.AnyParamType)


class ExtractHandleOp(TransformExtensionDialect.Operation, name="extract_handle"):
"""
Returns the handle at the specified index in `target`.
Args:
target: Handle(s) to target op(s)
target: Handle associated with operations or parameters.
index: Index of the handle to extract. Supports Python-style indexing.
silenceable: If False (default), an out-of-range index raises a
definite failure; if True, a silenceable failure.
Returns:
The handle at the specified index in `target`.
A handle of the same type as `target`, associated with its indexed element.
"""

target: ext.Operand[transform.AnyOpType]
target: ext.Operand[HandleType]
index: ext.Operand[transform.AnyParamType]
silenceable: ir.IntegerAttr = ext.attribute(
default_factory=lambda: ir.IntegerAttr.get(ir.IntegerType.get_signless(1), 0)
)
ops: ext.Result[transform.AnyOpType[()]] = ext.infer_result()
ops: ext.Result[HandleType] = ext.result(default_factory=transform.AnyOpType.get)

@classmethod
def attach_interface_impls(cls, ctx=None):
Expand All @@ -38,14 +44,19 @@ def apply(
results: transform.TransformResults,
state: transform.TransformState,
) -> DiagnosedSilenceableFailure:
target_ops = state.get_payload_ops(op.target)
is_param = isinstance(op.target.type, transform.AnyParamType)
targets = (
state.get_params(op.target)
if is_param
else state.get_payload_ops(op.target)
)
index_attr = state.get_params(op.index)
if len(index_attr) == 1 and isinstance(index_attr[0], ir.IntegerAttr):
index = index_attr[0].value
else:
return DiagnosedSilenceableFailure.SilenceableFailure

n = len(target_ops)
n = len(targets)
if index >= n or index < -n:
msg = (
f"extract_handle: invalid index {index} for target of length {n}. "
Expand All @@ -56,8 +67,11 @@ def apply(
return DiagnosedSilenceableFailure.SilenceableFailure
op.location.emit_error(msg)
return DiagnosedSilenceableFailure.DefiniteFailure
handle = target_ops[index]
results.set_ops(op.ops, [handle])
handle = targets[index]
if is_param:
results.set_params(op.ops, [handle])
else:
results.set_ops(op.ops, [handle])
return DiagnosedSilenceableFailure.Success

@staticmethod
Expand All @@ -75,27 +89,29 @@ def get_effects(op: ir.Operation):


def extract_handle(
target: ir.Value[transform.AnyOpType],
target: ir.Value[HandleType],
index: int | ir.Value[transform.AnyParamType],
silenceable: bool = False,
) -> ir.Value:
) -> ir.Value[HandleType]:
"""
snake_case wrapper to create a ExtractHandleOp.
Args:
target: Handle(s) to target op(s)
target: Handle associated with operations or parameters.
index: Index of the handle to extract. Supports Python-style indexing.
silenceable: If False (default), an out-of-range index raises a
definite failure; if True, a silenceable failure.
Returns:
The handle at the specified index in `target`.
A handle of the same type as `target`, associated with its indexed element.
"""
target = get_op_result_or_value(target)
if isinstance(index, int):
param_attr = ir.IntegerAttr.get(ir.IntegerType.get_signless(64), index)
index = transform.ParamConstantOp(transform.AnyParamType.get(), param_attr)

return ExtractHandleOp(
target=target,
ops=target.type,
index=index,
silenceable=ir.IntegerAttr.get(
ir.IntegerType.get_signless(1), int(silenceable)
Expand Down
Loading
Loading