|
| 1 | +Metric,avg,min,max,sum,p1,p5,p10,p25,p50,p75,p90,p95,p99,std |
| 2 | +Active Decode Throughput (tokens/sec),2571.54,0.00,227537.26,,,,,,2732.27,,3658.34,3858.31,4225.54,1004.11 |
| 3 | +Active Decode Throughput Per User (tokens/sec/user),65.28,0.00,5417.55,,,,,,72.03,,81.25,83.24,87.84,21.09 |
| 4 | +Active Prefill Throughput (tokens/sec),176704.89,430.81,588903.84,,,,,,172067.74,,249357.83,274605.55,324576.24,57383.26 |
| 5 | +Active Prefill Throughput Per User (tokens/sec/user),5279.84,430.81,19868.74,,,,,,4567.05,,8601.57,10781.31,14209.53,2456.67 |
| 6 | +Active Total Throughput (tokens/sec),178143.54,87.86,591598.12,,,,,,174495.61,,251662.61,277428.39,327220.50,59049.56 |
| 7 | +Credit Drop Latency (ms),1.29,0.28,15.46,477.10,0.30,0.40,0.43,0.52,0.93,1.47,2.26,3.29,6.41,1.37 |
| 8 | +Credit-to-Start Latency (ms),0.91,0.27,15.67,3058.72,0.38,0.43,0.46,0.53,0.62,0.90,1.62,2.35,4.91,0.88 |
| 9 | +E2E Output Token Throughput (tokens/sec/user),29.80,0.12,83.95,100321.77,1.09,4.16,6.46,14.45,28.18,43.92,55.55,61.71,68.54,18.22 |
| 10 | +Effective Concurrency,74.84,0.00,99.00,,,,,,76.00,,88.00,92.00,95.00,14.81 |
| 11 | +Effective Decode Concurrency,39.43,0.00,65.00,,,,,,40.00,,52.00,54.00,58.00,10.66 |
| 12 | +Effective Decode Throughput (tokens/sec),2540.94,0.00,227537.26,,,,,,2720.02,,3654.89,3854.53,4224.65,1036.33 |
| 13 | +Effective Decode Throughput Per User (tokens/sec/user),64.50,0.00,5417.55,,,,,,71.91,,81.22,83.20,87.77,22.13 |
| 14 | +Effective Latency (CO-aware) (ms),20900.75,844.80,751871.00,70351925.48,1647.36,3270.19,4388.58,7539.55,15626.82,26703.92,41010.36,54527.27,103246.05,24067.43 |
| 15 | +Effective Prefill Concurrency,35.41,0.00,61.00,,,,,,38.00,,49.00,52.00,57.00,11.81 |
| 16 | +Effective Prefill Throughput (tokens/sec),173550.82,0.00,588903.84,,,,,,171234.18,,248535.19,274037.63,324161.65,61493.53 |
| 17 | +Effective Prefill Throughput Per User (tokens/sec/user),5185.60,0.00,19868.74,,,,,,4527.32,,8559.50,10705.01,14185.10,2533.02 |
| 18 | +Effective Total Throughput (tokens/sec),176091.76,0.00,591598.12,,,,,,173626.00,,251224.70,276841.81,326754.25,61708.93 |
| 19 | +HTTP Blocked (ms),0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00 |
| 20 | +HTTP Chunks Received,239.33,1.00,15954.00,805594.00,5.00,18.00,35.00,54.00,112.00,248.00,554.00,823.00,1870.90,467.13 |
| 21 | +HTTP Chunks Sent,1.00,1.00,1.00,3366.00,1.00,1.00,1.00,1.00,1.00,1.00,1.00,1.00,1.00,0.00 |
| 22 | +HTTP Connecting (ms),0.02,0.00,9.90,67.12,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.42,0.21 |
| 23 | +HTTP Connection Overhead (ms),0.03,0.00,9.90,84.58,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.42,0.25 |
| 24 | +HTTP Connection Reused (ratio),0.97,0.00,1.00,3261.00,0.00,1.00,1.00,1.00,1.00,1.00,1.00,1.00,1.00,0.17 |
| 25 | +HTTP Data Received (KB),61.70,0.91,4066.95,207674.26,1.93,5.23,9.61,14.52,29.28,64.07,141.84,210.82,478.44,119.13 |
| 26 | +HTTP Data Sent (KB),189.13,0.58,470.46,636608.54,1.65,11.26,28.78,108.10,185.63,264.85,344.26,389.05,438.20,110.74 |
| 27 | +HTTP DNS Lookup (ms),0.01,0.00,1.43,17.46,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.08 |
| 28 | +HTTP Duration (excl. conn) (ms),20899.79,844.24,751870.44,70348701.08,1646.77,3269.56,4388.08,7538.96,15626.18,26702.54,41008.41,54526.20,103245.21,24067.30 |
| 29 | +HTTP Receiving (ms),11011.18,0.00,737767.25,37063637.10,163.70,756.64,1549.63,2485.11,5057.27,11309.94,25592.92,38960.05,85233.37,21611.80 |
| 30 | +HTTP Sending (ms),0.13,0.06,10.03,449.22,0.08,0.10,0.10,0.11,0.11,0.12,0.13,0.14,0.54,0.21 |
| 31 | +HTTP Total Time (ms),20899.82,844.24,751870.44,70348785.67,1646.77,3269.56,4388.08,7538.96,15626.18,26702.54,41008.41,54526.43,103245.21,24067.29 |
| 32 | +HTTP Waiting (TTFB) (ms),9888.48,357.27,105366.44,33284614.76,805.95,1125.67,1368.18,2371.88,5692.82,15013.37,23525.62,28506.34,41916.87,9961.21 |
| 33 | +Input Sequence Length (tokens),48466.43,150.00,118133.00,163137987.00,408.00,2964.00,7350.00,27539.25,47593.50,67918.50,88579.50,99699.50,112195.95,28412.75 |
| 34 | +Inter Chunk Latency (ms),46.20,0.00,28481.40,37061736.34,31.38,35.04,36.51,37.59,40.72,42.94,50.64,96.95,156.59,39.11 |
| 35 | +Inter Token Latency (ms),16.85,4.92,53.55,56710.64,11.32,12.94,13.66,14.87,16.46,18.31,20.43,21.82,26.17,3.02 |
| 36 | +OSL Mismatch Diff % (%),0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00 |
| 37 | +Output Sequence Length (tokens),710.59,1.00,52108.00,2391853.00,11.00,45.00,92.00,141.00,307.50,700.75,1640.00,2558.00,6027.80,1510.76 |
| 38 | +Output Token Count (tokens),710.59,1.00,52108.00,2391853.00,11.00,45.00,92.00,141.00,307.50,700.75,1640.00,2558.00,6027.80,1510.76 |
| 39 | +Output Token Throughput Per User (tokens/sec/user),61.10,18.68,203.19,205589.65,38.21,45.84,48.94,54.62,60.74,67.24,73.23,77.29,88.37,10.65 |
| 40 | +Prefill Throughput Per User (tokens/sec/user),10677.43,8.19,72719.53,35940239.78,120.51,773.13,1458.32,2959.03,5065.82,14879.85,27871.67,36730.63,52373.11,11787.07 |
| 41 | +Request Latency (ms),20899.45,844.30,751870.49,70347553.30,1646.81,3269.61,4388.12,7539.01,15626.23,26702.59,41008.46,54526.24,103245.26,24067.42 |
| 42 | +Requested OSL (tokens),710.59,1.00,52108.00,2391853.00,11.00,45.00,92.00,141.00,307.50,700.75,1640.00,2558.00,6027.80,1510.76 |
| 43 | +Time to First Output Token (ms),16980.64,1641.79,98928.47,19052282.05,2383.08,3585.16,4647.13,7752.27,14508.28,23434.53,31926.76,40198.53,54011.71,11979.80 |
| 44 | +Time to First Token (ms),9888.83,357.47,105366.62,33285816.96,806.11,1126.57,1368.34,2372.04,5692.98,15021.31,23525.79,28506.49,41917.04,9961.18 |
| 45 | +Time to Second Token (ms),40.38,0.00,234.96,135891.20,0.22,0.40,0.58,10.21,31.45,63.14,92.68,110.21,150.77,36.93 |
| 46 | +Tokens In Flight (tokens),4043844.42,0.00,5911716.39,,,,,,4432884.07,,5187818.57,5332813.30,5595634.28,1180166.36 |
| 47 | +Usage Completion Tokens (tokens),710.59,1.00,52108.00,2391853.00,11.00,45.00,92.00,141.00,307.50,700.75,1640.00,2558.00,6027.80,1510.76 |
| 48 | +Usage Prompt Cache Read Tokens (tokens),0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00 |
| 49 | +Usage Prompt Tokens (tokens),48466.43,150.00,118133.00,163137987.00,408.00,2964.00,7350.00,27539.25,47593.50,67918.50,88579.50,99699.50,112195.95,28412.75 |
| 50 | +Usage Total Tokens (tokens),49177.02,197.00,118854.00,165529840.00,463.35,3759.00,8029.00,28123.00,48245.00,68597.75,89326.00,100717.75,113377.30,28478.58 |
| 51 | + |
| 52 | +Metric,Value |
| 53 | +Benchmark Duration (sec),929.17 |
| 54 | +Input Token Throughput (tokens/sec),173550.82 |
| 55 | +Maximum Response Timestamp (ns),1785613977081609728.00 |
| 56 | +Minimum Request Timestamp (ns),1785613047906897664.00 |
| 57 | +OSL Mismatch Count,0.00 |
| 58 | +Output Token Throughput (tokens/sec),2544.52 |
| 59 | +Request Count,3366.00 |
| 60 | +Request Throughput (requests/sec),3.58 |
| 61 | +Theoretical Prefix Cache Hit (%),94.27 |
| 62 | +Total Input Sequence Length (tokens),163137987.00 |
| 63 | +Total Output Sequence Length (tokens),2391853.00 |
| 64 | +Total Output Tokens (tokens),2391853.00 |
| 65 | +Total Token Throughput (tokens/sec),176095.34 |
| 66 | +Total Usage Completion Tokens (tokens),2391853.00 |
| 67 | +Total Usage Prompt Cache Read Tokens (tokens),0.00 |
| 68 | +Total Usage Prompt Tokens (tokens),163137987.00 |
| 69 | +Total Usage Total Tokens (tokens),165529840.00 |
0 commit comments