|
| 1 | +# RUN: %PYTHON %S/kernel_bench.py --input %s --dump-kernel xegpu-wg | FileCheck %s --implicit-check-not=memref.alloc --implicit-check-not=memref.copy --implicit-check-not=gpu.alloc --implicit-check-not=math.log |
| 2 | +# REQUIRES: torch, torch_mlir |
| 3 | + |
| 4 | +"""Complete cache-free Llama 3.2 1B decoder layer for fixed-shape prefill. |
| 5 | +
|
| 6 | +The common XeGPU pipeline composes shared input RMSNorm, Q/K/V projections, |
| 7 | +RoPE, causal GQA, output projection/residual, and post-attention RMSNorm plus |
| 8 | +SwiGLU/residual. Weights are random; there is no KV cache, padding mask or dropout. |
| 9 | +The default schedule uses twelve kernels and nine intermediate allocations. |
| 10 | +Head merging and both residual additions are fused, without scratch copies. |
| 11 | +""" |
| 12 | + |
| 13 | +import torch |
| 14 | +import torch.nn as nn |
| 15 | +import torch.nn.functional as F |
| 16 | + |
| 17 | +from llama_rope import get_rope_tables, rotate_half |
| 18 | + |
| 19 | +batch_size = 4 |
| 20 | +n_ctx = 512 |
| 21 | +dim = 2048 |
| 22 | +hidden_dim = 8192 |
| 23 | +num_heads = 32 |
| 24 | +num_kv_heads = 8 |
| 25 | +d_head = 64 |
| 26 | + |
| 27 | + |
| 28 | +class RMSNorm(nn.Module): |
| 29 | + def __init__(self): |
| 30 | + super().__init__() |
| 31 | + self.weight = nn.Parameter(torch.ones(dim)) |
| 32 | + |
| 33 | + def forward(self, x: torch.Tensor) -> torch.Tensor: |
| 34 | + input_dtype = x.dtype |
| 35 | + x = x.float() |
| 36 | + variance = x.pow(2).mean(dim=-1, keepdim=True) |
| 37 | + return self.weight * (x * torch.rsqrt(variance + 1e-5)).to(input_dtype) |
| 38 | + |
| 39 | + |
| 40 | +class Model(nn.Module): |
| 41 | + """Cache-free Llama 3.2 1B decoder layer for fixed-shape prefill. |
| 42 | +
|
| 43 | + Inputs are hidden states [batch, sequence, 2048] and precomputed cos/sin |
| 44 | + tables [sequence, 64], shared across the batch. Output has the hidden-state |
| 45 | + shape. All projections are bias-free, with randomly initialized weights. |
| 46 | + RoPE table generation is outside the imported graph, as in HuggingFace's |
| 47 | + decoder layer. There is no padding mask, KV cache, or dropout. |
| 48 | + """ |
| 49 | + |
| 50 | + def __init__(self): |
| 51 | + super().__init__() |
| 52 | + self.input_layernorm = RMSNorm() |
| 53 | + self.q_proj = nn.Linear(dim, num_heads * d_head, bias=False) |
| 54 | + self.k_proj = nn.Linear(dim, num_kv_heads * d_head, bias=False) |
| 55 | + self.v_proj = nn.Linear(dim, num_kv_heads * d_head, bias=False) |
| 56 | + self.o_proj = nn.Linear(num_heads * d_head, dim, bias=False) |
| 57 | + self.post_attention_layernorm = RMSNorm() |
| 58 | + self.gate_proj = nn.Linear(dim, hidden_dim, bias=False) |
| 59 | + self.up_proj = nn.Linear(dim, hidden_dim, bias=False) |
| 60 | + self.down_proj = nn.Linear(hidden_dim, dim, bias=False) |
| 61 | + |
| 62 | + def forward( |
| 63 | + self, x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor |
| 64 | + ) -> torch.Tensor: |
| 65 | + batch, sequence, _ = x.shape |
| 66 | + h = self.input_layernorm(x) |
| 67 | + q = self.q_proj(h).view(batch, sequence, num_heads, d_head).transpose(1, 2) |
| 68 | + k = self.k_proj(h).view(batch, sequence, num_kv_heads, d_head).transpose(1, 2) |
| 69 | + v = self.v_proj(h).view(batch, sequence, num_kv_heads, d_head).transpose(1, 2) |
| 70 | + |
| 71 | + cos = cos[None, None, :, :] |
| 72 | + sin = sin[None, None, :, :] |
| 73 | + q = q * cos + rotate_half(q) * sin |
| 74 | + k = k * cos + rotate_half(k) * sin |
| 75 | + rows = torch.arange(sequence, device=x.device) |
| 76 | + allowed = rows[:, None] >= rows[None, :] |
| 77 | + # Avoid torch-mlir's SDPA XPU mask-factory and boolean-mask import issues. |
| 78 | + mask = allowed.to(dtype=x.dtype).log() |
| 79 | + attention = F.scaled_dot_product_attention( |
| 80 | + q, k, v, attn_mask=mask, is_causal=False, enable_gqa=True, dropout_p=0.0 |
| 81 | + ) |
| 82 | + attention = attention.transpose(1, 2).reshape(batch, sequence, dim) |
| 83 | + h = x + self.o_proj(attention) |
| 84 | + normalized = self.post_attention_layernorm(h) |
| 85 | + return h + self.down_proj( |
| 86 | + F.silu(self.gate_proj(normalized)) * self.up_proj(normalized) |
| 87 | + ) |
| 88 | + |
| 89 | + |
| 90 | +def get_inputs(): |
| 91 | + x = torch.randn(batch_size, n_ctx, dim) |
| 92 | + cos, sin = get_rope_tables(torch.arange(n_ctx)) |
| 93 | + return [x, cos, sin] |
| 94 | + |
| 95 | + |
| 96 | +def get_init_inputs(): |
| 97 | + return [] |
| 98 | + |
| 99 | + |
| 100 | +# CHECK-COUNT-9: gpu.alloc |
| 101 | +# CHECK-COUNT-6: gpu.func @main_kernel |
| 102 | +# CHECK: gpu.func @main_kernel |
| 103 | +# CHECK: arith.cmpi sge |
| 104 | +# CHECK-NOT: math.log |
| 105 | +# CHECK: xegpu.dpas |
| 106 | +# CHECK: math.exp |
| 107 | +# CHECK: xegpu.dpas |
| 108 | +# CHECK: xegpu.store_nd |
| 109 | +# CHECK: gpu.func @main_kernel |
| 110 | +# CHECK: xegpu.dpas |
| 111 | +# CHECK: arith.addf |
| 112 | +# CHECK: xegpu.store_nd |
| 113 | +# CHECK: gpu.func @main_kernel |
| 114 | +# CHECK: vector.multi_reduction |
| 115 | +# CHECK: math.rsqrt |
| 116 | +# CHECK-NOT: xegpu.dpas |
| 117 | +# CHECK: gpu.func @main_kernel |
| 118 | +# CHECK: xegpu.dpas |
| 119 | +# CHECK: math.exp |
| 120 | +# CHECK: xegpu.store_nd |
| 121 | +# CHECK: gpu.func @main_kernel |
| 122 | +# CHECK: xegpu.dpas |
| 123 | +# CHECK: arith.mulf |
| 124 | +# CHECK: xegpu.store_nd |
| 125 | +# CHECK: gpu.func @main_kernel |
| 126 | +# CHECK: xegpu.dpas |
| 127 | +# CHECK: arith.addf |
| 128 | +# CHECK: xegpu.store_nd |
| 129 | +# CHECK-NOT: gpu.func |
0 commit comments