|
| 1 | +# RUN: %PYTHON %S/kernel_bench.py --input %s --dump-kernel xegpu-wg | FileCheck %s --implicit-check-not=memref.alloc --implicit-check-not=memref.copy --implicit-check-not=math.log |
| 2 | +# REQUIRES: torch, torch_mlir |
| 3 | + |
| 4 | +"""Llama 3.2 1B prefill attention block, including output projection/residual. |
| 5 | +
|
| 6 | +Input RMSNorm is shared by Q/K/V projections. Q/K use half-split RoPE and |
| 7 | +causal grouped-query attention. Heads are merged before the bias-free output |
| 8 | +projection, and the original unnormalized hidden states supply the residual. |
| 9 | +There is no KV cache, padding mask, dropout, or feed-forward block. |
| 10 | +
|
| 11 | +The schedule uses eight GPU kernels. Head merging is fused into the attention |
| 12 | +store and the residual into the output projection. Seven intermediate tensors |
| 13 | +need six allocations because the dead Q-projection buffer is reused. |
| 14 | +""" |
| 15 | + |
| 16 | +import torch |
| 17 | +import torch.nn as nn |
| 18 | + |
| 19 | +from llama_rope import get_rope_tables, rotate_half |
| 20 | + |
| 21 | + |
| 22 | +batch_size = 4 |
| 23 | +n_ctx = 512 |
| 24 | +dim = 2048 |
| 25 | +num_heads = 32 |
| 26 | +num_kv_heads = 8 |
| 27 | +d_head = 64 |
| 28 | + |
| 29 | + |
| 30 | +class Model(nn.Module): |
| 31 | + def __init__(self): |
| 32 | + super().__init__() |
| 33 | + self.norm_weight = nn.Parameter(torch.ones(dim)) |
| 34 | + self.q_proj = nn.Linear(dim, num_heads * d_head, bias=False) |
| 35 | + self.k_proj = nn.Linear(dim, num_kv_heads * d_head, bias=False) |
| 36 | + self.v_proj = nn.Linear(dim, num_kv_heads * d_head, bias=False) |
| 37 | + self.o_proj = nn.Linear(num_heads * d_head, dim, bias=False) |
| 38 | + |
| 39 | + def forward( |
| 40 | + self, x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor |
| 41 | + ) -> torch.Tensor: |
| 42 | + batch, sequence, _ = x.shape |
| 43 | + h = x.float() |
| 44 | + variance = h.pow(2).mean(dim=-1, keepdim=True) |
| 45 | + h = self.norm_weight * (h * torch.rsqrt(variance + 1e-5)).to(x.dtype) |
| 46 | + q = self.q_proj(h).view(batch, sequence, num_heads, d_head).transpose(1, 2) |
| 47 | + k = self.k_proj(h).view(batch, sequence, num_kv_heads, d_head).transpose(1, 2) |
| 48 | + v = self.v_proj(h).view(batch, sequence, num_kv_heads, d_head).transpose(1, 2) |
| 49 | + cos = cos[None, None, :, :] |
| 50 | + sin = sin[None, None, :, :] |
| 51 | + q = q * cos + rotate_half(q) * sin |
| 52 | + k = k * cos + rotate_half(k) * sin |
| 53 | + rows = torch.arange(sequence, device=x.device) |
| 54 | + allowed = rows[:, None] >= rows[None, :] |
| 55 | + # Avoid SDPA's XPU mask-factory and boolean-mask import failures. |
| 56 | + mask = allowed.to(dtype=x.dtype).log() |
| 57 | + attention = torch.nn.functional.scaled_dot_product_attention( |
| 58 | + q, k, v, attn_mask=mask, is_causal=False, enable_gqa=True |
| 59 | + ) |
| 60 | + attention = attention.transpose(1, 2).reshape( |
| 61 | + batch, sequence, num_heads * d_head |
| 62 | + ) |
| 63 | + return x + self.o_proj(attention) |
| 64 | + |
| 65 | + |
| 66 | +def get_inputs(): |
| 67 | + x = torch.randn(batch_size, n_ctx, dim) |
| 68 | + cos, sin = get_rope_tables(torch.arange(n_ctx)) |
| 69 | + return [x, cos, sin] |
| 70 | + |
| 71 | + |
| 72 | +def get_init_inputs(): |
| 73 | + return [] |
| 74 | + |
| 75 | + |
| 76 | +# CHECK-COUNT-6: gpu.alloc |
| 77 | +# CHECK-NOT: gpu.alloc |
| 78 | +# CHECK-COUNT-6: gpu.func @main_kernel |
| 79 | +# CHECK: gpu.func @main_kernel |
| 80 | +# CHECK: arith.cmpi sge |
| 81 | +# CHECK-NOT: math.log |
| 82 | +# CHECK: xegpu.dpas |
| 83 | +# CHECK: math.exp |
| 84 | +# CHECK: xegpu.dpas |
| 85 | +# CHECK: xegpu.store_nd |
| 86 | +# CHECK: gpu.func @main_kernel |
| 87 | +# CHECK: xegpu.dpas |
| 88 | +# CHECK: arith.addf |
| 89 | +# CHECK: xegpu.store_nd |
| 90 | +# CHECK-NOT: gpu.func |
0 commit comments