Skip to content

Commit 7d926f6

Browse files
committed
add llama-bench_1B_decoder benchmark
1 parent 9d19a7c commit 7d926f6

1 file changed

Lines changed: 129 additions & 0 deletions

File tree

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
# RUN: %PYTHON %S/kernel_bench.py --input %s --dump-kernel xegpu-wg | FileCheck %s --implicit-check-not=memref.alloc --implicit-check-not=memref.copy --implicit-check-not=gpu.alloc --implicit-check-not=math.log
2+
# REQUIRES: torch, torch_mlir
3+
4+
"""Complete cache-free Llama 3.2 1B decoder layer for fixed-shape prefill.
5+
6+
The common XeGPU pipeline composes shared input RMSNorm, Q/K/V projections,
7+
RoPE, causal GQA, output projection/residual, and post-attention RMSNorm plus
8+
SwiGLU/residual. Weights are random; there is no KV cache, padding mask or dropout.
9+
The default schedule uses twelve kernels and nine intermediate allocations.
10+
Head merging and both residual additions are fused, without scratch copies.
11+
"""
12+
13+
import torch
14+
import torch.nn as nn
15+
import torch.nn.functional as F
16+
17+
from llama_rope import get_rope_tables, rotate_half
18+
19+
batch_size = 4
20+
n_ctx = 512
21+
dim = 2048
22+
hidden_dim = 8192
23+
num_heads = 32
24+
num_kv_heads = 8
25+
d_head = 64
26+
27+
28+
class RMSNorm(nn.Module):
29+
def __init__(self):
30+
super().__init__()
31+
self.weight = nn.Parameter(torch.ones(dim))
32+
33+
def forward(self, x: torch.Tensor) -> torch.Tensor:
34+
input_dtype = x.dtype
35+
x = x.float()
36+
variance = x.pow(2).mean(dim=-1, keepdim=True)
37+
return self.weight * (x * torch.rsqrt(variance + 1e-5)).to(input_dtype)
38+
39+
40+
class Model(nn.Module):
41+
"""Cache-free Llama 3.2 1B decoder layer for fixed-shape prefill.
42+
43+
Inputs are hidden states [batch, sequence, 2048] and precomputed cos/sin
44+
tables [sequence, 64], shared across the batch. Output has the hidden-state
45+
shape. All projections are bias-free, with randomly initialized weights.
46+
RoPE table generation is outside the imported graph, as in HuggingFace's
47+
decoder layer. There is no padding mask, KV cache, or dropout.
48+
"""
49+
50+
def __init__(self):
51+
super().__init__()
52+
self.input_layernorm = RMSNorm()
53+
self.q_proj = nn.Linear(dim, num_heads * d_head, bias=False)
54+
self.k_proj = nn.Linear(dim, num_kv_heads * d_head, bias=False)
55+
self.v_proj = nn.Linear(dim, num_kv_heads * d_head, bias=False)
56+
self.o_proj = nn.Linear(num_heads * d_head, dim, bias=False)
57+
self.post_attention_layernorm = RMSNorm()
58+
self.gate_proj = nn.Linear(dim, hidden_dim, bias=False)
59+
self.up_proj = nn.Linear(dim, hidden_dim, bias=False)
60+
self.down_proj = nn.Linear(hidden_dim, dim, bias=False)
61+
62+
def forward(
63+
self, x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor
64+
) -> torch.Tensor:
65+
batch, sequence, _ = x.shape
66+
h = self.input_layernorm(x)
67+
q = self.q_proj(h).view(batch, sequence, num_heads, d_head).transpose(1, 2)
68+
k = self.k_proj(h).view(batch, sequence, num_kv_heads, d_head).transpose(1, 2)
69+
v = self.v_proj(h).view(batch, sequence, num_kv_heads, d_head).transpose(1, 2)
70+
71+
cos = cos[None, None, :, :]
72+
sin = sin[None, None, :, :]
73+
q = q * cos + rotate_half(q) * sin
74+
k = k * cos + rotate_half(k) * sin
75+
rows = torch.arange(sequence, device=x.device)
76+
allowed = rows[:, None] >= rows[None, :]
77+
# Avoid torch-mlir's SDPA XPU mask-factory and boolean-mask import issues.
78+
mask = allowed.to(dtype=x.dtype).log()
79+
attention = F.scaled_dot_product_attention(
80+
q, k, v, attn_mask=mask, is_causal=False, enable_gqa=True, dropout_p=0.0
81+
)
82+
attention = attention.transpose(1, 2).reshape(batch, sequence, dim)
83+
h = x + self.o_proj(attention)
84+
normalized = self.post_attention_layernorm(h)
85+
return h + self.down_proj(
86+
F.silu(self.gate_proj(normalized)) * self.up_proj(normalized)
87+
)
88+
89+
90+
def get_inputs():
91+
x = torch.randn(batch_size, n_ctx, dim)
92+
cos, sin = get_rope_tables(torch.arange(n_ctx))
93+
return [x, cos, sin]
94+
95+
96+
def get_init_inputs():
97+
return []
98+
99+
100+
# CHECK-COUNT-9: gpu.alloc
101+
# CHECK-COUNT-6: gpu.func @main_kernel
102+
# CHECK: gpu.func @main_kernel
103+
# CHECK: arith.cmpi sge
104+
# CHECK-NOT: math.log
105+
# CHECK: xegpu.dpas
106+
# CHECK: math.exp
107+
# CHECK: xegpu.dpas
108+
# CHECK: xegpu.store_nd
109+
# CHECK: gpu.func @main_kernel
110+
# CHECK: xegpu.dpas
111+
# CHECK: arith.addf
112+
# CHECK: xegpu.store_nd
113+
# CHECK: gpu.func @main_kernel
114+
# CHECK: vector.multi_reduction
115+
# CHECK: math.rsqrt
116+
# CHECK-NOT: xegpu.dpas
117+
# CHECK: gpu.func @main_kernel
118+
# CHECK: xegpu.dpas
119+
# CHECK: math.exp
120+
# CHECK: xegpu.store_nd
121+
# CHECK: gpu.func @main_kernel
122+
# CHECK: xegpu.dpas
123+
# CHECK: arith.mulf
124+
# CHECK: xegpu.store_nd
125+
# CHECK: gpu.func @main_kernel
126+
# CHECK: xegpu.dpas
127+
# CHECK: arith.addf
128+
# CHECK: xegpu.store_nd
129+
# CHECK-NOT: gpu.func

0 commit comments

Comments
 (0)