Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions awscli/customizations/s3/factory.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
from awscli.compat import urlparse
from awscli.customizations.s3 import constants
from awscli.customizations.s3.transferconfig import (
DEFAULTS,
create_transfer_config_from_runtime_config,
)

Expand Down Expand Up @@ -157,16 +158,27 @@ def _create_crt_client(self, params, runtime_config):
return create_s3_crt_client(**create_crt_client_kwargs)

def _resolve_crt_client_config_kwargs(self, runtime_config):
use_defaults = self._should_use_transfer_config_defaults(
runtime_config
)
kwargs = {}
for config_name, crt_name in CRT_CLIENT_KWARG_MAP.items():
if runtime_config.is_explicitly_set(config_name):
kwargs[crt_name] = runtime_config[config_name]
elif use_defaults:
kwargs[crt_name] = DEFAULTS[config_name]
if 'part_size' not in kwargs:
# `create_s3_crt_client` defaults this to 8MB, so `None` has to be
# passed to opt into the CRT's dynamic part size calculation.
kwargs['part_size'] = None
return kwargs

def _should_use_transfer_config_defaults(self, runtime_config):
preferred = runtime_config.get('preferred_transfer_client')
if preferred == constants.CRT_TRANSFER_CLIENT:
return False
return not awscrt.s3.is_optimized_for_system()

def _create_crt_request_serializer(self, params):
return BotocoreCRTRequestSerializer(
self._session,
Expand Down
65 changes: 65 additions & 0 deletions tests/unit/customizations/s3/test_factory.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
from s3transfer.crt import CRTTransferManager
from s3transfer.manager import TransferManager

from awscli.customizations.s3 import constants
from awscli.customizations.s3.factory import (
ClientFactory,
TransferManagerFactory,
Expand Down Expand Up @@ -603,6 +604,70 @@ def test_max_concurrent_requests_configure_for_crt_manager(
mock_crt_client.call_args[1]['max_active_connections_override'], 3
)

def test_optimized_system_does_not_use_transfer_config_defaults(self):
runtime_config = self.get_runtime_config()
with mock.patch(
'awscrt.s3.is_optimized_for_system', return_value=True
):
self.assertFalse(
self.factory._should_use_transfer_config_defaults(
runtime_config
)
)

def test_explicit_crt_does_not_use_transfer_config_defaults(self):
runtime_config = self.get_runtime_config(
preferred_transfer_client='crt'
)
with mock.patch(
'awscrt.s3.is_optimized_for_system', return_value=False
):
self.assertFalse(
self.factory._should_use_transfer_config_defaults(
runtime_config
)
)

def test_newly_eligible_system_uses_transfer_config_defaults(self):
runtime_config = self.get_runtime_config()
with mock.patch(
'awscrt.s3.is_optimized_for_system', return_value=False
):
self.assertTrue(
self.factory._should_use_transfer_config_defaults(
runtime_config
)
)

@mock.patch('awscrt.s3.is_optimized_for_system', return_value=False)
@mock.patch('s3transfer.crt.S3Client')
def test_transfer_config_defaults_passed_for_newly_eligible_system(
self, mock_crt_client, mock_is_optimized
):
self.runtime_config = self.get_runtime_config()
with mock.patch.object(
self.factory,
'_resolve_transfer_client_type_for_system',
return_value=constants.CRT_TRANSFER_CLIENT,
):
transfer_manager = self.factory.create_transfer_manager(
self.params, self.runtime_config
)
self.assert_is_crt_manager(transfer_manager)
call_kwargs = mock_crt_client.call_args[1]
defaults = RuntimeConfig.defaults()
self.assertEqual(
call_kwargs['part_size'], defaults['multipart_chunksize']
)
self.assertEqual(
call_kwargs['multipart_upload_threshold'],
defaults['multipart_threshold'],
)
self.assertEqual(
call_kwargs['max_active_connections_override'],
defaults['max_concurrent_requests'],
)

@mock.patch('s3transfer.crt.S3Client')
def test_unconfigured_options_not_passed_to_crt_manager(
self, mock_crt_client
Expand Down
Loading