|
| 1 | +# AWS CLI Performance Benchmarks |
| 2 | + |
| 3 | +This document outlines details of the AWS CLI performance benchmarks, |
| 4 | +including how to run benchmarks and how to add your own. |
| 5 | + |
| 6 | +## Running the Benchmarks |
| 7 | + |
| 8 | +Our benchmark executor works by running all benchmarks defined in |
| 9 | +`benchmarks.json`. For each benchmark defined in this JSON file, it |
| 10 | +runs the command for a configurable number of iterations (default: 1) |
| 11 | +and benchmarks metrics such as memory usage, CPU utilization, and |
| 12 | +timings. |
| 13 | + |
| 14 | +The benchmark executor also stubs an HTTP client with mock responses |
| 15 | +defined in `benchmarks.json`. This ensures the timings produced in |
| 16 | +the results reflect only the AWS CLI and **not** external factors |
| 17 | +such as service latency or network throughput. |
| 18 | + |
| 19 | +### Example |
| 20 | + |
| 21 | +The following example command runs the benchmarks defined in `benchmarks.json`, |
| 22 | +and executes each command 2 times. |
| 23 | + |
| 24 | +`./run-benchmarks --num-iterations 2` |
| 25 | + |
| 26 | +An example output for this command is shown below. |
| 27 | + |
| 28 | +```json |
| 29 | +{ |
| 30 | + "results":[ |
| 31 | + { |
| 32 | + "name":"s3.cp.upload", |
| 33 | + "dimensions":[ |
| 34 | + { |
| 35 | + "FileSize":"32MB" |
| 36 | + }, |
| 37 | + { |
| 38 | + "S3TransferClient":"Classic" |
| 39 | + } |
| 40 | + ], |
| 41 | + "measurements":[ |
| 42 | + { |
| 43 | + "total_time":0.2531106472015381, |
| 44 | + "max_memory":76791808.0, |
| 45 | + "max_cpu":5.0, |
| 46 | + "p50_memory":51412992.0, |
| 47 | + "p95_memory":75235328.0, |
| 48 | + "p50_cpu":1.5, |
| 49 | + "p95_cpu":2.4, |
| 50 | + "first_client_invocation_time":0.24789667129516602 |
| 51 | + }, |
| 52 | + { |
| 53 | + "total_time":0.17595314979553223, |
| 54 | + "max_memory":76939264.0, |
| 55 | + "max_cpu":6.2, |
| 56 | + "p50_memory":52297728.0, |
| 57 | + "p95_memory":75710464.0, |
| 58 | + "p50_cpu":2.1, |
| 59 | + "p95_cpu":2.5, |
| 60 | + "first_client_invocation_time":0.17173004150390625 |
| 61 | + } |
| 62 | + ] |
| 63 | + }, |
| 64 | + { |
| 65 | + "name":"s3.cp.upload", |
| 66 | + "dimensions":[ |
| 67 | + { |
| 68 | + "FileSize":"32MB" |
| 69 | + }, |
| 70 | + { |
| 71 | + "S3TransferClient":"CRT" |
| 72 | + } |
| 73 | + ], |
| 74 | + "measurements":[ |
| 75 | + { |
| 76 | + "total_time":0.7724411487579346, |
| 77 | + "max_memory":81002496.0, |
| 78 | + "max_cpu":4.1, |
| 79 | + "p50_memory":78479360.0, |
| 80 | + "p95_memory":80822272.0, |
| 81 | + "p50_cpu":0.0, |
| 82 | + "p95_cpu":2.4, |
| 83 | + "first_client_invocation_time":0.17360806465148926 |
| 84 | + }, |
| 85 | + { |
| 86 | + "total_time":0.6735439300537109, |
| 87 | + "max_memory":80658432.0, |
| 88 | + "max_cpu":5.2, |
| 89 | + "p50_memory":78495744.0, |
| 90 | + "p95_memory":80412672.0, |
| 91 | + "p50_cpu":0.0, |
| 92 | + "p95_cpu":2.4, |
| 93 | + "first_client_invocation_time":0.17362713813781738 |
| 94 | + } |
| 95 | + ] |
| 96 | + }, |
| 97 | + { |
| 98 | + "name":"s3.mv.upload", |
| 99 | + "dimensions":[ |
| 100 | + { |
| 101 | + "FileSize":"32MB" |
| 102 | + } |
| 103 | + ], |
| 104 | + "measurements":[ |
| 105 | + { |
| 106 | + "total_time":0.17440271377563477, |
| 107 | + "max_memory":76972032.0, |
| 108 | + "max_cpu":4.6, |
| 109 | + "p50_memory":52166656.0, |
| 110 | + "p95_memory":75776000.0, |
| 111 | + "p50_cpu":2.1, |
| 112 | + "p95_cpu":2.5, |
| 113 | + "first_client_invocation_time":0.16981887817382812 |
| 114 | + }, |
| 115 | + { |
| 116 | + "total_time":0.17231082916259766, |
| 117 | + "max_memory":75825152.0, |
| 118 | + "max_cpu":6.1, |
| 119 | + "p50_memory":52199424.0, |
| 120 | + "p95_memory":74842112.0, |
| 121 | + "p50_cpu":2.1, |
| 122 | + "p95_cpu":2.5, |
| 123 | + "first_client_invocation_time":0.16803598403930664 |
| 124 | + } |
| 125 | + ] |
| 126 | + }, |
| 127 | + { |
| 128 | + "name":"s3.mv.download", |
| 129 | + "dimensions":[ |
| 130 | + { |
| 131 | + "FileSize":"32MB" |
| 132 | + }, |
| 133 | + { |
| 134 | + "S3TransferClient":"Classic" |
| 135 | + } |
| 136 | + ], |
| 137 | + "measurements":[ |
| 138 | + { |
| 139 | + "total_time":0.17304229736328125, |
| 140 | + "max_memory":76152832.0, |
| 141 | + "max_cpu":4.0, |
| 142 | + "p50_memory":52674560.0, |
| 143 | + "p95_memory":74907648.0, |
| 144 | + "p50_cpu":2.1, |
| 145 | + "p95_cpu":2.4, |
| 146 | + "first_client_invocation_time":0.16739511489868164 |
| 147 | + }, |
| 148 | + { |
| 149 | + "total_time":0.16962409019470215, |
| 150 | + "max_memory":76693504.0, |
| 151 | + "max_cpu":4.9, |
| 152 | + "p50_memory":52314112.0, |
| 153 | + "p95_memory":75431936.0, |
| 154 | + "p50_cpu":2.1, |
| 155 | + "p95_cpu":2.6, |
| 156 | + "first_client_invocation_time":0.16400408744812012 |
| 157 | + } |
| 158 | + ] |
| 159 | + }, |
| 160 | + { |
| 161 | + "name":"s3.sync.upload", |
| 162 | + "dimensions":[ |
| 163 | + { |
| 164 | + "FileCount":"5,000" |
| 165 | + }, |
| 166 | + { |
| 167 | + "FileSize":"4KB" |
| 168 | + }, |
| 169 | + { |
| 170 | + "S3TransferClient":"Classic" |
| 171 | + } |
| 172 | + ], |
| 173 | + "measurements":[ |
| 174 | + { |
| 175 | + "total_time":11.370934963226318, |
| 176 | + "max_memory":134578176.0, |
| 177 | + "max_cpu":20.7, |
| 178 | + "p50_memory":106397696.0, |
| 179 | + "p95_memory":132235264.0, |
| 180 | + "p50_cpu":2.4, |
| 181 | + "p95_cpu":2.7, |
| 182 | + "first_client_invocation_time":0.6362888813018799 |
| 183 | + }, |
| 184 | + { |
| 185 | + "total_time":12.029011964797974, |
| 186 | + "max_memory":134676480.0, |
| 187 | + "max_cpu":18.6, |
| 188 | + "p50_memory":105955328.0, |
| 189 | + "p95_memory":131727360.0, |
| 190 | + "p50_cpu":2.4, |
| 191 | + "p95_cpu":2.7, |
| 192 | + "first_client_invocation_time":0.6395571231842041 |
| 193 | + } |
| 194 | + ] |
| 195 | + }, |
| 196 | + { |
| 197 | + "name":"s3.sync.upload", |
| 198 | + "dimensions":[ |
| 199 | + { |
| 200 | + "FileCount":"5,000" |
| 201 | + }, |
| 202 | + { |
| 203 | + "FileSize":"4KB" |
| 204 | + }, |
| 205 | + { |
| 206 | + "S3TransferClient":"CRT" |
| 207 | + } |
| 208 | + ], |
| 209 | + "measurements":[ |
| 210 | + { |
| 211 | + "total_time":90.28388690948486, |
| 212 | + "max_memory":188809216.0, |
| 213 | + "max_cpu":17.9, |
| 214 | + "p50_memory":144375808.0, |
| 215 | + "p95_memory":188792832.0, |
| 216 | + "p50_cpu":0.0, |
| 217 | + "p95_cpu":3.4, |
| 218 | + "first_client_invocation_time":0.656865119934082 |
| 219 | + }, |
| 220 | + { |
| 221 | + "total_time":84.99997591972351, |
| 222 | + "max_memory":190808064.0, |
| 223 | + "max_cpu":20.7, |
| 224 | + "p50_memory":143917056.0, |
| 225 | + "p95_memory":186728448.0, |
| 226 | + "p50_cpu":0.0, |
| 227 | + "p95_cpu":3.5, |
| 228 | + "first_client_invocation_time":0.7549021244049072 |
| 229 | + } |
| 230 | + ] |
| 231 | + } |
| 232 | + ] |
| 233 | +} |
| 234 | +``` |
| 235 | + |
| 236 | +## Defining Your own Benchmarks for Local Performance Testing |
| 237 | + |
| 238 | +To create your own benchmark definitions, create a file on your machine containing |
| 239 | +a JSON-formatted list of benchmark definitions. Each benchmark definition supports |
| 240 | +the keys below. Each key is required unless specified otherwise. |
| 241 | + |
| 242 | +- `name` (string): The name of the benchmark. |
| 243 | +- `command` (list): The AWS CLI command to benchmark, including arguments. |
| 244 | + - Each element of the list is a string component of the command. |
| 245 | + - Example value: `["s3", "cp", "test_file", "s3://bucket/test_file", "--quiet"]`. |
| 246 | +- `dimensions` (list) **(optional)**: Used to specify additional dimensions for |
| 247 | +interpreting this metric. |
| 248 | + - Each element in the list is an object with a single key-value pair. |
| 249 | +The key is the name of the dimension (e.g. `FileSize`), and the value |
| 250 | +is the value of the dimension (e.g. `32MB`). |
| 251 | +- `environment` (object) **(optional)**: Specifies settings for the environment to run |
| 252 | +the command in. |
| 253 | + - The environment object supports the following keys: |
| 254 | + - `file_literals` (list) **(optional)**: Specifies files that must be |
| 255 | +created before executing the benchmark. The files created will contain |
| 256 | +the specified contents. |
| 257 | + - Each element is an object with the following keys: |
| 258 | + - `name` (string): Name of the file to create |
| 259 | + - `content` (string): The contents of the file. |
| 260 | + - `files` (list) **(optional)**: Specifies the files that must be |
| 261 | +created before executing the benchmark. The files created will be filled with |
| 262 | +null bytes to achieve the specified size. |
| 263 | + - Each element is an object with the following keys: |
| 264 | + - `name` (string): Name of the file to create |
| 265 | + - `size` (int): The size of the file to create in bytes. |
| 266 | + - `file_dirs` (list) **(optional)**: Specifies the directories that must |
| 267 | +be created before executing the benchmark. The directories will be created |
| 268 | +and filled with the specified number of files, each of which will be filled |
| 269 | +with null bytes to achieve the specified file size. |
| 270 | + - Each element is an object with the following keys: |
| 271 | + - `name` (string): Name of the directory |
| 272 | + - `file_count` (int): The number of files to create in the directory. |
| 273 | + - `file_size` (int): The size of each file in the directory, in bytes. |
| 274 | + - `config` (string) **(optional)**: The contents of the AWS config |
| 275 | +file to use for the benchmark execution. |
| 276 | + - Default: `"[default]"`. |
| 277 | + - Example value: `"[default]\ns3 =\n preferred_transfer_client = crt"` |
| 278 | +- `responses` (list) **(optional)**: A list of HTTP responses to stub from |
| 279 | +the service for each request made during command execution. |
| 280 | + - Default: `[{{"headers": {}, "body": ""}]` |
| 281 | + - Each element of the list is an object with the following keys: |
| 282 | + - `status_code` (int) **(optional)**: The status code of the response. |
| 283 | + - Default: `200` |
| 284 | + - `headers` (object) **(optional)**: Used to specify the HTTP headers of |
| 285 | +the response. Each key-value pair corresponds to a single header name (key) |
| 286 | +and its value. |
| 287 | + - Default: `{}` |
| 288 | + - `body` (string) **(optional)**: The raw HTTP response. |
| 289 | + - Default: `""` |
| 290 | + - `instances` (int) **(optional)**: The total number of times to stub |
| 291 | +this response; this prevents the need to repeat the same response many times. |
| 292 | + - Default: 1 |
| 293 | + - This is useful for commands such as `aws s3 sync`, that may execute many |
| 294 | + HTTP requests with similar responses. |
0 commit comments