Skip to content

Commit 9d19a7c

Browse files
committed
add llama-bench_1B_attention_block benchmark
1 parent 6e2c146 commit 9d19a7c

1 file changed

Lines changed: 90 additions & 0 deletions

File tree

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
# RUN: %PYTHON %S/kernel_bench.py --input %s --dump-kernel xegpu-wg | FileCheck %s --implicit-check-not=memref.alloc --implicit-check-not=memref.copy --implicit-check-not=math.log
2+
# REQUIRES: torch, torch_mlir
3+
4+
"""Llama 3.2 1B prefill attention block, including output projection/residual.
5+
6+
Input RMSNorm is shared by Q/K/V projections. Q/K use half-split RoPE and
7+
causal grouped-query attention. Heads are merged before the bias-free output
8+
projection, and the original unnormalized hidden states supply the residual.
9+
There is no KV cache, padding mask, dropout, or feed-forward block.
10+
11+
The schedule uses eight GPU kernels. Head merging is fused into the attention
12+
store and the residual into the output projection. Seven intermediate tensors
13+
need six allocations because the dead Q-projection buffer is reused.
14+
"""
15+
16+
import torch
17+
import torch.nn as nn
18+
19+
from llama_rope import get_rope_tables, rotate_half
20+
21+
22+
batch_size = 4
23+
n_ctx = 512
24+
dim = 2048
25+
num_heads = 32
26+
num_kv_heads = 8
27+
d_head = 64
28+
29+
30+
class Model(nn.Module):
31+
def __init__(self):
32+
super().__init__()
33+
self.norm_weight = nn.Parameter(torch.ones(dim))
34+
self.q_proj = nn.Linear(dim, num_heads * d_head, bias=False)
35+
self.k_proj = nn.Linear(dim, num_kv_heads * d_head, bias=False)
36+
self.v_proj = nn.Linear(dim, num_kv_heads * d_head, bias=False)
37+
self.o_proj = nn.Linear(num_heads * d_head, dim, bias=False)
38+
39+
def forward(
40+
self, x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor
41+
) -> torch.Tensor:
42+
batch, sequence, _ = x.shape
43+
h = x.float()
44+
variance = h.pow(2).mean(dim=-1, keepdim=True)
45+
h = self.norm_weight * (h * torch.rsqrt(variance + 1e-5)).to(x.dtype)
46+
q = self.q_proj(h).view(batch, sequence, num_heads, d_head).transpose(1, 2)
47+
k = self.k_proj(h).view(batch, sequence, num_kv_heads, d_head).transpose(1, 2)
48+
v = self.v_proj(h).view(batch, sequence, num_kv_heads, d_head).transpose(1, 2)
49+
cos = cos[None, None, :, :]
50+
sin = sin[None, None, :, :]
51+
q = q * cos + rotate_half(q) * sin
52+
k = k * cos + rotate_half(k) * sin
53+
rows = torch.arange(sequence, device=x.device)
54+
allowed = rows[:, None] >= rows[None, :]
55+
# Avoid SDPA's XPU mask-factory and boolean-mask import failures.
56+
mask = allowed.to(dtype=x.dtype).log()
57+
attention = torch.nn.functional.scaled_dot_product_attention(
58+
q, k, v, attn_mask=mask, is_causal=False, enable_gqa=True
59+
)
60+
attention = attention.transpose(1, 2).reshape(
61+
batch, sequence, num_heads * d_head
62+
)
63+
return x + self.o_proj(attention)
64+
65+
66+
def get_inputs():
67+
x = torch.randn(batch_size, n_ctx, dim)
68+
cos, sin = get_rope_tables(torch.arange(n_ctx))
69+
return [x, cos, sin]
70+
71+
72+
def get_init_inputs():
73+
return []
74+
75+
76+
# CHECK-COUNT-6: gpu.alloc
77+
# CHECK-NOT: gpu.alloc
78+
# CHECK-COUNT-6: gpu.func @main_kernel
79+
# CHECK: gpu.func @main_kernel
80+
# CHECK: arith.cmpi sge
81+
# CHECK-NOT: math.log
82+
# CHECK: xegpu.dpas
83+
# CHECK: math.exp
84+
# CHECK: xegpu.dpas
85+
# CHECK: xegpu.store_nd
86+
# CHECK: gpu.func @main_kernel
87+
# CHECK: xegpu.dpas
88+
# CHECK: arith.addf
89+
# CHECK: xegpu.store_nd
90+
# CHECK-NOT: gpu.func

0 commit comments

Comments
 (0)