Merge pull request #39 from cbrichford/botocore-updates

Botocore updates

aiobotocore/client.py

@@ -7,62 +7,14 @@
 import botocore.serialize
 import botocore.validate
 import botocore.parsers
-from botocore.exceptions import ClientError, OperationNotPageableError, \
-    ParamValidationError
+from botocore.exceptions import ClientError, OperationNotPageableError
 from botocore.paginate import Paginator
+from botocore.client import ClientEndpointBridge
 from botocore.signers import RequestSigner
 
 from .paginate import AioPageIterator
 from .endpoint import AioEndpointCreator
-
-
-class AioConfig(botocore.client.Config):
-
-    def __init__(self, connector_args=None, **kwargs):
-        super().__init__(**kwargs)
-
-        self._validate_connector_args(connector_args)
-        self.connector_args = copy.copy(connector_args)
-        if not self.connector_args:
-            self.connector_args = dict()
-
-        if 'keepalive_timeout' not in self.connector_args:
-            # AWS has a 20 second idle timeout:
-            # https://forums.aws.amazon.com/message.jspa?messageID=215367
-            # and aiohttp default timeout is 30s so we set it to something
-            # reasonable here
-            self.connector_args['keepalive_timeout'] = 12
-
-    @staticmethod
-    def _validate_connector_args(connector_args):
-        if connector_args is None:
-            return
-
-        for k, v in connector_args.items():
-            if k in ['use_dns_cache', 'verify_ssl']:
-                if not isinstance(v, bool):
-                    raise ParamValidationError(
-                        report='{} value must be a boolean'.format(k))
-            elif k in ['conn_timeout', 'keepalive_timeout']:
-                if not isinstance(v, float) and not isinstance(v, int):
-                    raise ParamValidationError(
-                        report='{} value must be a float/int'.format(k))
-            elif k == 'force_close':
-                if not isinstance(v, bool):
-                    raise ParamValidationError(
-                        report='{} value must be a boolean'.format(k))
-            elif k == 'limit':
-                if not isinstance(v, int):
-                    raise ParamValidationError(
-                        report='{} value must be an int'.format(k))
-            elif k == 'ssl_context':
-                import ssl
-                if not isinstance(v, ssl.SSLContext):
-                    raise ParamValidationError(
-                        report='{} must be an SSLContext instance'.format(k))
-            else:
-                raise ParamValidationError(
-                    report='invalid connector_arg:{}'.format(k))
+from .config import AioConfig
 
 
 class AioClientCreator(botocore.client.ClientCreator):
@@ -80,32 +32,25 @@ def _get_client_args(self, service_model, region_name, is_secure,
                          endpoint_url, verify, credentials,
                          scoped_config, client_config):
 
+        # This is a near copy of botocore.client.ClientCreator. What's replaced
+        # is Config->AioConfig and EndpointCreator->AioEndpointCreator
+        # We don't re-use the parent's implementations due to weak refs
+
+        service_name = service_model.endpoint_prefix
         protocol = service_model.metadata['protocol']
+        parameter_validation = True
+        if client_config:
+            parameter_validation = client_config.parameter_validation
         serializer = botocore.serialize.create_serializer(
-            protocol, include_validation=True)
+            protocol, parameter_validation)
 
         event_emitter = copy.copy(self._event_emitter)
-
         response_parser = botocore.parsers.create_parser(protocol)
-
-        # Determine what region the user provided either via the
-        # region_name argument or the client_config.
-        if region_name is None:
-            if client_config and client_config.region_name is not None:
-                region_name = client_config.region_name
-
-        # Based on what the user provided use the scoped config file
-        # to determine if the region is going to change and what
-        # signature should be used.
-        signature_version, region_name = \
-            self._get_signature_version_and_region(
-                service_model, region_name, is_secure, scoped_config,
-                endpoint_url)
-
-        # Override the signature if the user specifies it in the client
-        # config.
-        if client_config and client_config.signature_version is not None:
-            signature_version = client_config.signature_version
+        endpoint_bridge = ClientEndpointBridge(
+            self._endpoint_resolver, scoped_config, client_config,
+            service_signing_name=service_model.metadata.get('signingName'))
+        endpoint_config = endpoint_bridge.resolve(
+            service_name, region_name, endpoint_url, is_secure)
 
         # Override the user agent if specified in the client config.
         user_agent = self._user_agent
@@ -115,36 +60,38 @@ def _get_client_args(self, service_model, region_name, is_secure,
             if client_config.user_agent_extra is not None:
                 user_agent += ' %s' % client_config.user_agent_extra
 
-        signer = RequestSigner(service_model.service_name, region_name,
-                               service_model.signing_name,
-                               signature_version, credentials,
-                               event_emitter)
+        signer = RequestSigner(
+            service_name, endpoint_config['signing_region'],
+            endpoint_config['signing_name'],
+            endpoint_config['signature_version'],
+            credentials, event_emitter)
 
         # Create a new client config to be passed to the client based
         # on the final values. We do not want the user to be able
         # to try to modify an existing client with a client config.
         config_kwargs = dict(
-            region_name=region_name, signature_version=signature_version,
+            region_name=endpoint_config['region_name'],
+            signature_version=endpoint_config['signature_version'],
             user_agent=user_agent)
-
         if client_config is not None:
             config_kwargs.update(
                 connect_timeout=client_config.connect_timeout,
                 read_timeout=client_config.read_timeout)
 
-            if isinstance(client_config, AioConfig):
-                config_kwargs.update(
-                    connector_args=client_config.connector_args)
+        # Add any additional s3 configuration for client
+        self._inject_s3_configuration(
+            config_kwargs, scoped_config, client_config)
 
-        new_config = AioConfig(**config_kwargs)
-
-        endpoint_creator = AioEndpointCreator(self._endpoint_resolver,
-                                              region_name, event_emitter,
-                                              self._loop)
+        if isinstance(client_config, AioConfig):
+            connector_args = client_config.connector_args
+        else:
+            connector_args = None
 
+        new_config = AioConfig(connector_args, **config_kwargs)
+        endpoint_creator = AioEndpointCreator(event_emitter, self._loop)
         endpoint = endpoint_creator.create_endpoint(
-            service_model, region_name, is_secure=is_secure,
-            endpoint_url=endpoint_url, verify=verify,
+            service_model, region_name=endpoint_config['region_name'],
+            endpoint_url=endpoint_config['endpoint_url'], verify=verify,
             response_parser_factory=self._response_parser_factory,
             timeout=(new_config.connect_timeout, new_config.read_timeout),
             connector_args=new_config.connector_args)

aiobotocore/config.py

@@ -0,0 +1,58 @@
+import botocore.client
+import copy
+from botocore.exceptions import ParamValidationError
+
+
+class AioConfig(botocore.client.Config):
+
+    def __init__(self, connector_args=None, **kwargs):
+        super().__init__(**kwargs)
+
+        self._validate_connector_args(connector_args)
+        self.connector_args = copy.copy(connector_args)
+        if not self.connector_args:
+            self.connector_args = dict()
+
+        if 'keepalive_timeout' not in self.connector_args:
+            # AWS has a 20 second idle timeout:
+            # https://forums.aws.amazon.com/message.jspa?messageID=215367
+            # and aiohttp default timeout is 30s so we set it to something
+            # reasonable here
+            self.connector_args['keepalive_timeout'] = 12
+
+    def merge(self, other_config):
+        # Adapted from parent class
+        config_options = copy.copy(self._user_provided_options)
+        config_options.update(other_config._user_provided_options)
+        return AioConfig(self.connector_args, **config_options)
+
+    @staticmethod
+    def _validate_connector_args(connector_args):
+        if connector_args is None:
+            return
+
+        for k, v in connector_args.items():
+            if k in ['use_dns_cache', 'verify_ssl']:
+                if not isinstance(v, bool):
+                    raise ParamValidationError(
+                        report='{} value must be a boolean'.format(k))
+            elif k in ['conn_timeout', 'keepalive_timeout']:
+                if not isinstance(v, float) and not isinstance(v, int):
+                    raise ParamValidationError(
+                        report='{} value must be a float/int'.format(k))
+            elif k == 'force_close':
+                if not isinstance(v, bool):
+                    raise ParamValidationError(
+                        report='{} value must be a boolean'.format(k))
+            elif k == 'limit':
+                if not isinstance(v, int):
+                    raise ParamValidationError(
+                        report='{} value must be an int'.format(k))
+            elif k == 'ssl_context':
+                import ssl
+                if not isinstance(v, ssl.SSLContext):
+                    raise ParamValidationError(
+                        report='{} must be an SSLContext instance'.format(k))
+            else:
+                raise ParamValidationError(
+                    report='invalid connector_arg:{}'.format(k))

aiobotocore/endpoint.py

@@ -5,8 +5,7 @@
 
 from botocore.utils import is_valid_endpoint_url
 from botocore.endpoint import EndpointCreator, Endpoint, DEFAULT_TIMEOUT
-from botocore.exceptions import EndpointConnectionError, \
-    BaseEndpointResolverError
+from botocore.exceptions import EndpointConnectionError
 
 PY_35 = sys.version_info >= (3, 5)
 
@@ -191,49 +190,24 @@ def _get_response(self, request, operation_model, attempts):
 
 class AioEndpointCreator(EndpointCreator):
 
-    def __init__(self, endpoint_resolver, configured_region, event_emitter,
-                 loop):
-        super().__init__(endpoint_resolver, configured_region, event_emitter)
+    def __init__(self, event_emitter, loop):
+        super().__init__(event_emitter)
         self._loop = loop
 
-    def create_endpoint(self, service_model, region_name=None, is_secure=True,
+    def create_endpoint(self, service_model, region_name=None,
                         endpoint_url=None, verify=None,
                         response_parser_factory=None, timeout=DEFAULT_TIMEOUT,
                         connector_args=None):
-        if region_name is None:
-            region_name = self._configured_region
-        # Use the endpoint resolver heuristics to build the endpoint url.
-        scheme = 'https' if is_secure else 'http'
-        try:
-            endpoint = self._endpoint_resolver.construct_endpoint(
-                service_model.endpoint_prefix,
-                region_name, scheme=scheme)
-        except BaseEndpointResolverError:
-            if endpoint_url is not None:
-                # If the user provides an endpoint_url, it's ok
-                # if the heuristics didn't find anything.  We use the
-                # user provided endpoint_url.
-                endpoint = {'uri': endpoint_url, 'properties': {}}
-            else:
-                raise
 
-        if endpoint_url is not None:
-            # If the user provides an endpoint url, we'll use that
-            # instead of what the heuristics rule gives us.
-            final_endpoint_url = endpoint_url
-        else:
-            final_endpoint_url = endpoint['uri']
-        if not is_valid_endpoint_url(final_endpoint_url):
-            raise ValueError("Invalid endpoint: %s" % final_endpoint_url)
+        if not is_valid_endpoint_url(endpoint_url):
+            raise ValueError("Invalid endpoint: %s" % endpoint_url)
 
-        proxies = self._get_proxies(final_endpoint_url)
-        verify_value = self._get_verify_value(verify)
         return AioEndpoint(
-            final_endpoint_url,
+            endpoint_url,
             endpoint_prefix=service_model.endpoint_prefix,
             event_emitter=self._event_emitter,
-            proxies=proxies,
-            verify=verify_value,
+            proxies=self._get_proxies(endpoint_url),
+            verify=self._get_verify_value(verify),
             timeout=timeout,
             response_parser_factory=response_parser_factory, loop=self._loop,
             connector_args=connector_args)

aiobotocore/paginate.py

@@ -9,52 +9,18 @@
 
 class AioPageIterator(PageIterator):
 
-    def __init__(self, method, input_token, output_token, more_results,
-                 result_keys, non_aggregate_keys, limit_key, max_items,
-                 starting_token, page_size, op_kwargs):
-        self._method = method
-        self._op_kwargs = op_kwargs
-        self._input_token = input_token
-        self._output_token = output_token
-        self._more_results = more_results
-        self._result_keys = result_keys
-        self._max_items = max_items
-        self._limit_key = limit_key
-        self._starting_token = starting_token
-        self._page_size = page_size
-        self._op_kwargs = op_kwargs
-        self._resume_token = None
-        self._non_aggregate_key_exprs = non_aggregate_keys
-        self._non_aggregate_part = {}
-
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
         self._init_pager()
 
-    @property
-    def result_keys(self):
-        return self._result_keys
-
-    @property
-    def resume_token(self):
-        """Token to specify to resume pagination."""
-        return self._resume_token
-
-    @resume_token.setter
-    def resume_token(self, value):
-        if isinstance(value, list):
-            self._resume_token = '___'.join([str(v) for v in value])
-
-    @property
-    def non_aggregate_part(self):
-        return self._non_aggregate_part
-
     def __iter__(self):
         raise NotImplementedError
 
     def _init_pager(self):
         self._is_stop = False
         self._current_kwargs = self._op_kwargs
         self._previous_next_token = None
-        self._next_token = [None for _ in range(len(self._input_token))]
+        self._next_token = dict((key, None) for key in self._input_token)
         # The number of items from result_key we've seen so far.
         self._total_items = 0
         self._first_request = True

aiobotocore/session.py

@@ -21,11 +21,24 @@ def create_client(self, service_name, region_name=None, api_version=None,
                       aws_access_key_id=None, aws_secret_access_key=None,
                       aws_session_token=None, config=None):
 
+        default_client_config = self.get_default_client_config()
+
+        if config is not None and default_client_config is not None:
+            config = default_client_config.merge(config)
+        elif default_client_config is not None:
+            config = default_client_config
+
         if region_name is None:
             if config and config.region_name is not None:
                 region_name = config.region_name
             else:
                 region_name = self.get_config_variable('region')
+
+        # Figure out the verify value base on the various
+        # configuration options.
+        if verify is None:
+            verify = self.get_config_variable('ca_bundle')
+
         loader = self.get_component('data_loader')
         event_emitter = self.get_component('event_emitter')
         response_parser_factory = self.get_component(
@@ -38,6 +51,7 @@ def create_client(self, service_name, region_name=None, api_version=None,
         else:
             credentials = self.get_credentials()
         endpoint_resolver = self.get_component('endpoint_resolver')
+
         client_creator = AioClientCreator(
             loader, endpoint_resolver, self.user_agent(), event_emitter,
             retryhandler, translate, response_parser_factory, loop=self._loop)

setup.py

@@ -4,7 +4,7 @@
 from setuptools import setup, find_packages
 
 
-install_requires = ['botocore==1.3.26', 'aiohttp>=0.21.2']
+install_requires = ['botocore>=1.4.0', 'aiohttp>=0.21.2']
 
 PY_VER = sys.version_info