From efc11546c9a7f744dd0a5ccbdb7dcf2eaf60f433 Mon Sep 17 00:00:00 2001 From: Steve Yoo Date: Tue, 19 Aug 2025 10:51:01 -0400 Subject: [PATCH 1/2] Expose io_chunksize S3 transfer config --- .../next-release/enhancement-s3-32921.json | 5 +++++ awscli/customizations/s3/transferconfig.py | 4 +++- awscli/topics/s3-config.rst | 22 +++++++++++++++++++ tests/unit/customizations/s3/test_factory.py | 2 ++ .../customizations/s3/test_transferconfig.py | 2 ++ 5 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 .changes/next-release/enhancement-s3-32921.json diff --git a/.changes/next-release/enhancement-s3-32921.json b/.changes/next-release/enhancement-s3-32921.json new file mode 100644 index 000000000000..a5b4fc8f33f1 --- /dev/null +++ b/.changes/next-release/enhancement-s3-32921.json @@ -0,0 +1,5 @@ +{ + "type": "enhancement", + "category": "``s3``", + "description": "Expose S3 transfer config's ``io_chunksize`` in shared config file" +} diff --git a/awscli/customizations/s3/transferconfig.py b/awscli/customizations/s3/transferconfig.py index c5b7a0474800..dd695c0541fe 100644 --- a/awscli/customizations/s3/transferconfig.py +++ b/awscli/customizations/s3/transferconfig.py @@ -45,8 +45,9 @@ class RuntimeConfig: 'max_queue_size', 'max_bandwidth', 'target_bandwidth', + 'io_chunksize', ] - HUMAN_READABLE_SIZES = ['multipart_chunksize', 'multipart_threshold'] + HUMAN_READABLE_SIZES = ['multipart_chunksize', 'multipart_threshold', 'io_chunksize'] HUMAN_READABLE_RATES = ['max_bandwidth', 'target_bandwidth'] SUPPORTED_CHOICES = { 'preferred_transfer_client': [ @@ -206,6 +207,7 @@ def create_transfer_config_from_runtime_config(runtime_config): 'multipart_threshold': 'multipart_threshold', 'multipart_chunksize': 'multipart_chunksize', 'max_bandwidth': 'max_bandwidth', + 'io_chunksize': 'io_chunksize', } kwargs = {} for key, value in runtime_config.items(): diff --git a/awscli/topics/s3-config.rst b/awscli/topics/s3-config.rst index 6c0ef3de0934..c7979d230bcc 100644 --- a/awscli/topics/s3-config.rst +++ b/awscli/topics/s3-config.rst @@ -30,6 +30,8 @@ command set: size that the CLI uses for multipart transfers of individual files. * ``max_bandwidth`` - The maximum bandwidth that will be consumed for uploading and downloading data to and from Amazon S3. +* ``io_chunksize`` - The maximum size of read parts that can be queued in-memory + to be written for a download. For experimental ``s3`` configuration values, see the the `Experimental Configuration Values <#experimental-configuration-values>`__ @@ -208,6 +210,26 @@ threads having to wait unnecessarily which can lead to excess resource consumption and connection timeouts. +io_chunksize +------------ + +**Default** - ``256KB`` + +When a GET request is called for downloads, the response contains a file-like +object that streams data fetched from S3. Chunks are read from the stream and +queued in-memory for writes. ``io_chunksize`` configures the maximum size of +elements in the IO queue. This value can be specified using the same semantics +as ``multipart_threshold``, that is either as the number of bytes as an +integer, or using a size suffix. + +Increasing this value may result in higher overall throughput by preventing +blocking in cases where large objects are downloaded in environments where +network speed exceeds disk write speed. It is recommended to only configure +``io_chunksize`` if overall download throughput is constrained by writes. +In cases where network IO is the bottleneck, it is recommended to configure +``max_concurrent_requests`` instead. + + use_accelerate_endpoint ----------------------- diff --git a/tests/unit/customizations/s3/test_factory.py b/tests/unit/customizations/s3/test_factory.py index b79897850890..f153fec5de12 100644 --- a/tests/unit/customizations/s3/test_factory.py +++ b/tests/unit/customizations/s3/test_factory.py @@ -231,6 +231,7 @@ def test_proxies_transfer_config_to_default_transfer_manager(self): max_concurrent_requests=20, max_queue_size=5000, max_bandwidth=10 * MB, + io_chunksize=1 * MB, ) transfer_manager = self.factory.create_transfer_manager( self.params, self.runtime_config @@ -240,6 +241,7 @@ def test_proxies_transfer_config_to_default_transfer_manager(self): self.assertEqual(transfer_manager.config.max_request_concurrency, 20) self.assertEqual(transfer_manager.config.max_request_queue_size, 5000) self.assertEqual(transfer_manager.config.max_bandwidth, 10 * MB) + self.assertEqual(transfer_manager.config.io_chunksize, 1 * MB) # These configurations are hardcoded and not configurable but # we just want to make sure they are being set by the factory. self.assertEqual( diff --git a/tests/unit/customizations/s3/test_transferconfig.py b/tests/unit/customizations/s3/test_transferconfig.py index 56fc5be17c77..77e5ab46e9e7 100644 --- a/tests/unit/customizations/s3/test_transferconfig.py +++ b/tests/unit/customizations/s3/test_transferconfig.py @@ -149,6 +149,7 @@ def test_convert(self): 'max_bandwidth': 1024 * 1024, 'addressing_style': 'path', 'use_accelerate_endpoint': True, + 'io_chunksize': 1024 * 1024, # This is a TransferConfig only option, it should # just be ignored if it's in the ~/.aws/config for now. 'max_in_memory_upload_chunks': 1000, @@ -161,4 +162,5 @@ def test_convert(self): assert result.max_request_concurrency == 3 assert result.max_request_queue_size == 4 assert result.max_bandwidth == 1024 * 1024 + assert result.io_chunksize == 1024 * 1024 assert result.max_in_memory_upload_chunks != 1000 From 9d9a18924e792c38043ffafb45189b6d5d70ccfb Mon Sep 17 00:00:00 2001 From: Steve Yoo Date: Wed, 10 Sep 2025 10:40:32 -0400 Subject: [PATCH 2/2] Add config to CLI defaults --- awscli/customizations/s3/transferconfig.py | 1 + 1 file changed, 1 insertion(+) diff --git a/awscli/customizations/s3/transferconfig.py b/awscli/customizations/s3/transferconfig.py index dd695c0541fe..5227217d868c 100644 --- a/awscli/customizations/s3/transferconfig.py +++ b/awscli/customizations/s3/transferconfig.py @@ -30,6 +30,7 @@ 'max_bandwidth': None, 'preferred_transfer_client': constants.AUTO_RESOLVE_TRANSFER_CLIENT, 'target_bandwidth': None, + 'io_chunksize': 256 * 1024, }