|
| 1 | +Metric,avg,min,max,sum,p1,p5,p10,p25,p50,p75,p90,p95,p99,std |
| 2 | +Active Decode Throughput (tokens/sec),5819.73,56.27,257114.11,,,,,,5147.54,,9663.01,10262.09,11159.12,2787.07 |
| 3 | +Active Decode Throughput Per User (tokens/sec/user),36.39,8.75,1518.40,,,,,,37.18,,56.37,59.40,66.45,17.00 |
| 4 | +Active Prefill Throughput (tokens/sec),393064.13,4103.75,736134.30,,,,,,413768.50,,527689.45,552900.72,603821.67,118779.78 |
| 5 | +Active Prefill Throughput Per User (tokens/sec/user),3488.54,623.68,12649.40,,,,,,3702.70,,5304.80,5814.27,6526.88,1498.60 |
| 6 | +Active Total Throughput (tokens/sec),395604.95,36.27,741854.76,,,,,,418427.33,,533186.56,558613.52,609216.45,124009.79 |
| 7 | +Credit Drop Latency (ms),1.28,0.27,16.52,1259.66,0.32,0.41,0.44,0.52,0.94,1.50,2.37,3.31,6.09,1.27 |
| 8 | +Credit-to-Start Latency (ms),0.91,0.26,16.65,7775.05,0.35,0.42,0.44,0.51,0.61,0.97,1.72,2.39,4.73,0.87 |
| 9 | +E2E Output Token Throughput (tokens/sec/user),17.39,0.16,59.57,147896.04,0.75,2.99,4.68,9.41,16.56,24.48,30.51,34.25,43.86,10.01 |
| 10 | +Effective Concurrency,289.05,0.00,347.00,,,,,,299.00,,316.00,328.00,340.00,52.53 |
| 11 | +Effective Decode Concurrency,164.77,0.00,235.00,,,,,,175.00,,204.00,210.00,222.00,42.01 |
| 12 | +Effective Decode Throughput (tokens/sec),5751.98,0.00,257114.11,,,,,,5093.40,,9650.58,10254.53,11156.81,2840.26 |
| 13 | +Effective Decode Throughput Per User (tokens/sec/user),35.97,0.00,1518.40,,,,,,35.97,,56.30,59.36,66.39,17.34 |
| 14 | +Effective Latency (CO-aware) (ms),31954.77,1360.96,428022.84,271711434.10,3052.66,5996.48,7916.20,13371.22,23688.00,40218.87,62284.63,84053.80,149280.82,31465.76 |
| 15 | +Effective Prefill Concurrency,124.28,0.00,237.00,,,,,,117.00,,187.00,205.00,223.00,43.99 |
| 16 | +Effective Prefill Throughput (tokens/sec),385501.22,0.00,736134.30,,,,,,410506.98,,526699.46,552374.14,603241.09,129432.17 |
| 17 | +Effective Prefill Throughput Per User (tokens/sec/user),3421.41,0.00,12649.40,,,,,,3663.33,,5293.66,5803.91,6513.12,1559.57 |
| 18 | +Effective Total Throughput (tokens/sec),391253.19,0.00,741854.76,,,,,,416556.07,,532718.73,558161.69,609216.45,130045.76 |
| 19 | +HTTP Blocked (ms),0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00 |
| 20 | +HTTP Chunks Received,218.83,1.00,4750.00,1860678.00,5.00,20.00,36.00,56.00,113.00,243.00,504.00,748.80,1615.52,335.50 |
| 21 | +HTTP Chunks Sent,1.00,1.00,1.00,8503.00,1.00,1.00,1.00,1.00,1.00,1.00,1.00,1.00,1.00,0.00 |
| 22 | +HTTP Connecting (ms),0.02,0.00,11.99,145.31,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.34,0.26 |
| 23 | +HTTP Connection Overhead (ms),0.02,0.00,11.99,184.03,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.34,0.31 |
| 24 | +HTTP Connection Reused (ratio),0.98,0.00,1.00,8291.00,0.00,1.00,1.00,1.00,1.00,1.00,1.00,1.00,1.00,0.16 |
| 25 | +HTTP Data Received (KB),56.43,0.91,1197.43,479863.10,1.94,5.73,9.82,15.01,29.52,62.62,128.80,191.24,413.15,85.40 |
| 26 | +HTTP Data Sent (KB),165.84,0.59,459.17,1410121.41,1.60,6.31,11.94,73.99,164.50,247.09,314.24,367.99,415.14,111.18 |
| 27 | +HTTP DNS Lookup (ms),0.00,0.00,2.54,38.72,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.10 |
| 28 | +HTTP Duration (excl. conn) (ms),31953.81,1359.58,428020.86,271703249.30,3052.10,5995.80,7915.44,13370.54,23687.35,40218.20,62284.16,84052.30,149279.87,31465.61 |
| 29 | +HTTP Receiving (ms),18215.56,0.00,399966.39,154886916.88,332.69,1411.23,2709.52,4558.78,9295.26,20241.66,42096.09,63420.50,132910.55,28436.07 |
| 30 | +HTTP Sending (ms),0.13,0.06,12.08,1095.28,0.07,0.09,0.10,0.11,0.11,0.12,0.13,0.14,0.45,0.26 |
| 31 | +HTTP Total Time (ms),31953.83,1359.58,428020.86,271703433.33,3052.10,5995.80,7915.44,13370.54,23687.35,40218.20,62285.12,84052.33,149279.87,31465.62 |
| 32 | +HTTP Waiting (TTFB) (ms),13738.12,466.18,97222.89,116815237.15,1262.46,2113.03,2870.38,4156.87,9401.04,18972.31,30405.44,39933.60,59119.44,13011.91 |
| 33 | +Input Sequence Length (tokens),42616.94,151.00,117855.00,362371869.00,407.00,1624.00,2970.00,18768.50,42401.00,63555.50,80762.80,94601.60,106951.48,28631.63 |
| 34 | +Inter Chunk Latency (ms),83.62,0.00,197423.36,154876208.74,39.85,46.71,49.47,53.16,59.34,104.59,159.43,186.79,238.29,165.35 |
| 35 | +Inter Token Latency (ms),30.85,9.37,116.75,262309.94,16.93,20.25,22.50,26.27,30.31,34.63,39.26,42.71,52.81,7.35 |
| 36 | +OSL Mismatch Diff % (%),0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00 |
| 37 | +Output Sequence Length (tokens),636.88,1.00,16721.00,5415373.00,12.00,45.00,94.00,144.00,309.00,696.50,1473.00,2234.00,4985.78,1059.26 |
| 38 | +Output Token Count (tokens),636.88,1.00,16721.00,5415373.00,12.00,45.00,94.00,144.00,309.00,696.50,1473.00,2234.00,4985.78,1059.26 |
| 39 | +Output Token Throughput Per User (tokens/sec/user),34.16,8.57,106.70,290425.56,18.94,23.41,25.47,28.88,32.99,38.07,44.45,49.38,59.08,8.01 |
| 40 | +Prefill Throughput Per User (tokens/sec/user),6237.41,3.61,72902.12,53036689.03,24.82,118.95,314.10,1475.67,3628.23,8829.33,16082.39,20302.89,29793.80,6854.70 |
| 41 | +Request Latency (ms),31952.85,1359.63,428020.91,271695059.31,3052.15,5995.86,7912.13,13370.59,23687.40,40218.25,62284.21,84052.35,149279.92,31465.85 |
| 42 | +Requested OSL (tokens),636.88,1.00,16721.00,5415373.00,12.00,45.00,94.00,144.00,309.00,696.50,1473.00,2234.00,4985.78,1059.26 |
| 43 | +Time to First Output Token (ms),27049.60,2233.24,180323.01,73845399.37,4623.99,6923.20,8961.84,14184.07,23508.46,35556.28,49846.07,60211.50,86240.93,17493.60 |
| 44 | +Time to First Token (ms),13738.55,466.35,97223.05,116818850.57,1262.64,2115.60,2870.52,4157.04,9401.20,18972.48,30405.60,39933.82,59119.58,13011.90 |
| 45 | +Time to Second Token (ms),89.35,0.00,368.19,759681.19,1.36,17.21,29.88,48.47,85.23,122.07,158.03,181.55,229.08,51.47 |
| 46 | +Tokens In Flight (tokens),12994580.85,0.00,17504260.84,,,,,,14361918.03,,16438539.27,16855523.81,17286473.27,3746643.57 |
| 47 | +Usage Completion Tokens (tokens),636.88,1.00,16721.00,5415373.00,12.00,45.00,94.00,144.00,309.00,696.50,1473.00,2234.00,4985.78,1059.26 |
| 48 | +Usage Prompt Cache Read Tokens (tokens),0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00 |
| 49 | +Usage Prompt Tokens (tokens),42616.94,151.00,117855.00,362371869.00,407.00,1624.00,2970.00,18768.50,42401.00,63555.50,80762.80,94601.60,106951.48,28631.63 |
| 50 | +Usage Total Tokens (tokens),43253.82,169.00,119175.00,367787242.00,433.02,1975.30,4141.00,19350.00,42919.00,64271.50,81839.80,95417.00,108354.92,28696.65 |
| 51 | + |
| 52 | +Metric,Value |
| 53 | +Benchmark Duration (sec),929.66 |
| 54 | +Input Token Throughput (tokens/sec),385501.22 |
| 55 | +Maximum Response Timestamp (ns),1785653821881334016.00 |
| 56 | +Minimum Request Timestamp (ns),1785652892219712256.00 |
| 57 | +OSL Mismatch Count,0.00 |
| 58 | +Output Token Throughput (tokens/sec),5761.02 |
| 59 | +Request Count,8503.00 |
| 60 | +Request Throughput (requests/sec),9.05 |
| 61 | +Theoretical Prefix Cache Hit (%),93.98 |
| 62 | +Total Input Sequence Length (tokens),362371869.00 |
| 63 | +Total Output Sequence Length (tokens),5415373.00 |
| 64 | +Total Output Tokens (tokens),5415373.00 |
| 65 | +Total Token Throughput (tokens/sec),391262.24 |
| 66 | +Total Usage Completion Tokens (tokens),5415373.00 |
| 67 | +Total Usage Prompt Cache Read Tokens (tokens),0.00 |
| 68 | +Total Usage Prompt Tokens (tokens),362371869.00 |
| 69 | +Total Usage Total Tokens (tokens),367787242.00 |
0 commit comments