Skip to content

Commit 691d6e4

Browse files
nilignilig
committed
findings: weka campaign - asymmetric tiers, prefill 25 GiB/rank vs decode 100
Tier turnover measured on p1w1d1w1 shows the asymmetry running backwards: decode churns its tier 5.4x versus prefill's 1.4x and never restores locally, so the only possible source forgets faster than the router's choice. c64 and c128 on symmetric tiers confirm it (max remote advantage 16,256 then 6,592 tokens, 1 then 0 directives). Shrinking the prefill tier to 215 GB against decode's 859 GB forces the router's choice to lose history its peer still holds. Weights, workload, model and engine unchanged; both cells of each pair run on this deployment. Signed-off-by: nilig <nili.ifergan@gmail.com>
1 parent c14fea7 commit 691d6e4

22 files changed

Lines changed: 20061 additions & 24 deletions

‎test/p2p-findings/configs/glm-weka-paused-2p1d/artifacts/p1w1d1w1-c128-s43-blog-precise-p2p/profile_export_aiperf.json‎

Lines changed: 1766 additions & 0 deletions
Large diffs are not rendered by default.

‎test/p2p-findings/configs/glm-weka-paused-2p1d/artifacts/p1w1d1w1-c128-s43-blog-precise-p2p/profile_export_console.txt‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
2+
3+
NVIDIA AIPerf | LLM Metrics: Effective
4+
┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┓
5+
┃ Metric ┃ avg ┃ min ┃ max ┃ p99 ┃ p90 ┃ p50 ┃ std ┃
6+
┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━┩
7+
│ Effective Concurrency (requests) │ 136.27 │ 0.00 │ 170.00 │ 168.00 │ 162.00 │ 144.00 │ 32.00 │
8+
│ Effective Decode Throughput │ 1,466.11 │ 0.00 │ 80,502.49 │ 3,183.67 │ 2,522.97 │ 1,367.83 │ 674.03 │
9+
│ (tokens/sec) │ │ │ │ │ │ │ │
10+
│ Effective Prefill Throughput │ 67,054.35 │ 0.00 │ 163,387.87 │ 102,012.62 │ 83,411.21 │ 68,918.76 │ 17,430.28 │
11+
│ (tokens/sec) │ │ │ │ │ │ │ │
12+
│ Effective Decode Concurrency │ 25.89 │ 0.00 │ 62.00 │ 58.00 │ 45.00 │ 23.00 │ 11.60 │
13+
│ (requests) │ │ │ │ │ │ │ │
14+
│ Effective Prefill Concurrency │ 110.38 │ 0.00 │ 138.00 │ 135.00 │ 129.00 │ 119.00 │ 27.16 │
15+
│ (requests) │ │ │ │ │ │ │ │
16+
│ Effective Total Throughput │ 68,520.46 │ 0.00 │ 163,699.57 │ 103,145.61 │ 84,960.78 │ 70,382.55 │ 17,662.33 │
17+
│ (tokens/sec) │ │ │ │ │ │ │ │
18+
│ Effective Decode Throughput Per │ 57.09 │ 0.00 │ 5,138.55 │ 71.85 │ 66.30 │ 60.32 │ 14.20 │
19+
│ User │ │ │ │ │ │ │ │
20+
│ (tokens/sec/user) │ │ │ │ │ │ │ │
21+
│ Effective Prefill Throughput Per │ 625.34 │ 0.00 │ 10,905.73 │ 1,118.59 │ 748.74 │ 591.40 │ 367.91 │
22+
│ User │ │ │ │ │ │ │ │
23+
│ (tokens/sec/user) │ │ │ │ │ │ │ │
24+
│ Tokens In Flight (tokens) │ 5,322,962.82 │ 0.00 │ 6,766,384.37 │ 6,725,860.98 │ 6,524,286.26 │ 5,882,565.32 │ 1,432,808.82 │
25+
│ Effective Latency (CO-aware) (ms) │ 67,346.02 │ 1,746.11 │ 392,223.03 │ 242,035.78 │ 129,099.58 │ 55,870.13 │ 50,355.63 │
26+
└─────────────────────────────────────┴──────────────┴──────────┴──────────────┴──────────────┴──────────────┴──────────────┴──────────────┘
27+
NVIDIA AIPerf | LLM Metrics: Active
28+
┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━┳━━━━━━━━━━━━┳━━━━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━━┓
29+
┃ Metric ┃ avg ┃ min ┃ max ┃ p99 ┃ p90 ┃ p50 ┃ std ┃
30+
┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━╇━━━━━━━━━━━━╇━━━━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━━┩
31+
│ Active Decode Throughput (tokens/sec) │ 1,483.15 │ 0.00 │ 80,502.49 │ 3,186.61 │ 2,528.57 │ 1,374.53 │ 659.04 │
32+
│ Active Prefill Throughput (tokens/sec) │ 68,029.24 │ 792.09 │ 163,387.87 │ 102,059.12 │ 83,515.59 │ 69,062.87 │ 15,553.47 │
33+
│ Active Decode Throughput Per User │ 57.75 │ 0.00 │ 5,138.55 │ 71.88 │ 66.34 │ 60.38 │ 12.87 │
34+
│ (tokens/sec/user) │ │ │ │ │ │ │ │
35+
│ Active Prefill Throughput Per User │ 634.43 │ 418.33 │ 10,905.73 │ 1,119.14 │ 750.10 │ 593.10 │ 362.71 │
36+
│ (tokens/sec/user) │ │ │ │ │ │ │ │
37+
│ Active Total Throughput (tokens/sec) │ 69,282.91 │ 62.12 │ 163,699.57 │ 103,145.61 │ 84,979.78 │ 70,517.24 │ 16,205.07 │
38+
└────────────────────────────────────────┴───────────┴────────┴────────────┴────────────┴───────────┴───────────┴───────────┘
39+
NVIDIA AIPerf | LLM Metrics: Usage
40+
┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━┳━━━━━━━━━━━━┳━━━━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━━┓
41+
┃ Metric ┃ avg ┃ min ┃ max ┃ p99 ┃ p90 ┃ p50 ┃ std ┃
42+
┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━╇━━━━━━━━╇━━━━━━━━━━━━╇━━━━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━━┩
43+
│ Usage Prompt Tokens (tokens) │ 33,139.40 │ 149.00 │ 118,057.00 │ 104,132.73 │ 76,406.00 │ 26,922.50 │ 29,071.09 │
44+
│ Usage Prompt Cache Read Tokens (tokens) │ 0.00 │ 0.00 │ 0.00 │ 0.00 │ 0.00 │ 0.00 │ 0.00 │
45+
│ Usage Completion Tokens (tokens) │ 725.58 │ 1.00 │ 16,721.00 │ 6,438.60 │ 1,592.70 │ 329.00 │ 1,303.44 │
46+
│ Usage Total Tokens (tokens) │ 33,864.98 │ 166.00 │ 118,952.00 │ 104,593.83 │ 76,860.70 │ 27,384.50 │ 29,212.11 │
47+
│ Total Usage Prompt Tokens (tokens) │ 63,031,147.00 │ N/A │ N/A │ N/A │ N/A │ N/A │ N/A │
48+
│ Total Usage Prompt Cache Read Tokens │ 0.00 │ N/A │ N/A │ N/A │ N/A │ N/A │ N/A │
49+
│ (tokens) │ │ │ │ │ │ │ │
50+
│ Total Usage Completion Tokens (tokens) │ 1,380,049.00 │ N/A │ N/A │ N/A │ N/A │ N/A │ N/A │
51+
│ Total Usage Total Tokens (tokens) │ 64,411,196.00 │ N/A │ N/A │ N/A │ N/A │ N/A │ N/A │
52+
└─────────────────────────────────────────┴───────────────┴────────┴────────────┴────────────┴───────────┴───────────┴───────────┘
53+
NVIDIA AIPerf | LLM Metrics
54+
┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━━┳━━━━━━━━━━━━┳━━━━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━━┓
55+
┃ Metric ┃ avg ┃ min ┃ max ┃ p99 ┃ p90 ┃ p50 ┃ std ┃
56+
┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━━╇━━━━━━━━━━━━╇━━━━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━━┩
57+
│ Time to First Token (ms) │ 54,551.86 │ 450.76 │ 298,069.90 │ 222,665.38 │ 111,709.28 │ 44,212.61 │ 46,066.64 │
58+
│ Time to Second Token (ms) │ 84.09 │ 0.00 │ 27,083.09 │ 165.55 │ 96.71 │ 45.28 │ 825.64 │
59+
│ Time to First Output Token (ms) │ 64,923.55 │ 2,709.68 │ 299,643.89 │ 229,545.65 │ 120,686.78 │ 54,152.18 │ 44,840.14 │
60+
│ Request Latency (ms) │ 67,344.68 │ 1,745.57 │ 392,222.47 │ 242,035.28 │ 129,097.03 │ 55,869.26 │ 50,355.75 │
61+
│ Inter Token Latency (ms) │ 19.16 │ 8.91 │ 174.35 │ 31.94 │ 22.67 │ 18.53 │ 6.27 │
62+
│ Output Token Throughput Per User │ 54.15 │ 5.74 │ 112.20 │ 75.77 │ 64.49 │ 53.96 │ 8.83 │
63+
│ (tokens/sec/user) │ │ │ │ │ │ │ │
64+
│ E2E Output Token Throughput (tokens/sec/user) │ 12.82 │ 0.04 │ 62.18 │ 52.08 │ 33.93 │ 7.86 │ 13.19 │
65+
│ Output Sequence Length (tokens) │ 725.58 │ 1.00 │ 16,721.00 │ 6,438.60 │ 1,592.70 │ 329.00 │ 1,303.44 │
66+
│ Input Sequence Length (tokens) │ 33,139.40 │ 149.00 │ 118,057.00 │ 104,132.73 │ 76,406.00 │ 26,922.50 │ 29,071.09 │
67+
│ Output Token Throughput (tokens/sec) │ 1,468.14 │ N/A │ N/A │ N/A │ N/A │ N/A │ N/A │
68+
│ Input Token Throughput (tokens/sec) │ 67,054.35 │ N/A │ N/A │ N/A │ N/A │ N/A │ N/A │
69+
│ Request Throughput (requests/sec) │ 2.02 │ N/A │ N/A │ N/A │ N/A │ N/A │ N/A │
70+
│ Request Count (requests) │ 1,902.00 │ N/A │ N/A │ N/A │ N/A │ N/A │ N/A │
71+
│ Theoretical Prefix Cache Hit (%) │ 93.60 │ N/A │ N/A │ N/A │ N/A │ N/A │ N/A │
72+
└───────────────────────────────────────────────┴───────────┴──────────┴────────────┴────────────┴────────────┴───────────┴───────────┘

0 commit comments

Comments
 (0)