[v2] Expose io_chunksize S3 transfer config

.changes/next-release/enhancement-s3-32921.json

@@ -0,0 +1,5 @@
+{
+  "type": "enhancement",
+  "category": "``s3``",
+  "description": "Expose S3 transfer config's ``io_chunksize`` in shared config file"
+}

awscli/customizations/s3/transferconfig.py

@@ -30,6 +30,7 @@
     'max_bandwidth': None,
     'preferred_transfer_client': constants.AUTO_RESOLVE_TRANSFER_CLIENT,
     'target_bandwidth': None,
+    'io_chunksize': 256 * 1024,
 }
 
 
@@ -45,8 +46,9 @@ class RuntimeConfig:
         'max_queue_size',
         'max_bandwidth',
         'target_bandwidth',
+        'io_chunksize',
     ]
-    HUMAN_READABLE_SIZES = ['multipart_chunksize', 'multipart_threshold']
+    HUMAN_READABLE_SIZES = ['multipart_chunksize', 'multipart_threshold', 'io_chunksize']
     HUMAN_READABLE_RATES = ['max_bandwidth', 'target_bandwidth']
     SUPPORTED_CHOICES = {
         'preferred_transfer_client': [
@@ -206,6 +208,7 @@ def create_transfer_config_from_runtime_config(runtime_config):
         'multipart_threshold': 'multipart_threshold',
         'multipart_chunksize': 'multipart_chunksize',
         'max_bandwidth': 'max_bandwidth',
+        'io_chunksize': 'io_chunksize',
     }
     kwargs = {}
     for key, value in runtime_config.items():

awscli/topics/s3-config.rst

@@ -30,6 +30,8 @@ command set:
   size that the CLI uses for multipart transfers of individual files.
 * ``max_bandwidth`` - The maximum bandwidth that will be consumed for uploading
   and downloading data to and from Amazon S3.
+* ``io_chunksize`` - The maximum size of read parts that can be queued in-memory
+  to be written for a download.
 
 For experimental ``s3`` configuration values, see the the
 `Experimental Configuration Values <#experimental-configuration-values>`__
@@ -208,6 +210,26 @@ threads having to wait unnecessarily which can lead to excess resource
 consumption and connection timeouts.
 
 
+io_chunksize
+------------
+
+**Default** - ``256KB``
+
+When a GET request is called for downloads, the response contains a file-like
+object that streams data fetched from S3. Chunks are read from the stream and
+queued in-memory for writes. ``io_chunksize`` configures the maximum size of
+elements in the IO queue. This value can be specified using the same semantics
+as ``multipart_threshold``, that is either as the number of bytes as an
+integer, or using a size suffix.
+
+Increasing this value may result in higher overall throughput by preventing
+blocking in cases where large objects are downloaded in environments where
+network speed exceeds disk write speed. It is recommended to only configure
+``io_chunksize`` if overall download throughput is constrained by writes.
+In cases where network IO is the bottleneck, it is recommended to configure
+``max_concurrent_requests`` instead.
+
+
 use_accelerate_endpoint
 -----------------------
 

tests/unit/customizations/s3/test_factory.py

@@ -231,6 +231,7 @@ def test_proxies_transfer_config_to_default_transfer_manager(self):
             max_concurrent_requests=20,
             max_queue_size=5000,
             max_bandwidth=10 * MB,
+            io_chunksize=1 * MB,
         )
         transfer_manager = self.factory.create_transfer_manager(
             self.params, self.runtime_config
@@ -240,6 +241,7 @@ def test_proxies_transfer_config_to_default_transfer_manager(self):
         self.assertEqual(transfer_manager.config.max_request_concurrency, 20)
         self.assertEqual(transfer_manager.config.max_request_queue_size, 5000)
         self.assertEqual(transfer_manager.config.max_bandwidth, 10 * MB)
+        self.assertEqual(transfer_manager.config.io_chunksize, 1 * MB)
         # These configurations are hardcoded and not configurable but
         # we just want to make sure they are being set by the factory.
         self.assertEqual(

tests/unit/customizations/s3/test_transferconfig.py

@@ -149,6 +149,7 @@ def test_convert(self):
             'max_bandwidth': 1024 * 1024,
             'addressing_style': 'path',
             'use_accelerate_endpoint': True,
+            'io_chunksize': 1024 * 1024,
             # This is a TransferConfig only option, it should
             # just be ignored if it's in the ~/.aws/config for now.
             'max_in_memory_upload_chunks': 1000,
@@ -161,4 +162,5 @@ def test_convert(self):
         assert result.max_request_concurrency == 3
         assert result.max_request_queue_size == 4
         assert result.max_bandwidth == 1024 * 1024
+        assert result.io_chunksize == 1024 * 1024
         assert result.max_in_memory_upload_chunks != 1000