Support up-to-date botocore (#1057)

* bump botocore dependency specification

CHANGES.rst

@@ -1,11 +1,11 @@
 Changes
 -------
 
-2.8.0 (2023-10-25)
+2.8.0 (2023-11-28)
 ^^^^^^^^^^^^^^^^^^
 * add AioStubber that returns AioAWSResponse()
 * remove confusing `aiobotocore.session.Session` symbol
-* relax botocore dependency specification
+* bump botocore dependency specification
 
 2.7.0 (2023-10-17)
 ^^^^^^^^^^^^^^^^^^

aiobotocore/args.py

@@ -167,14 +167,16 @@ def _build_endpoint_resolver(
             client_endpoint_url=endpoint_url,
             legacy_endpoint_url=endpoint.host,
         )
-        # botocore does not support client context parameters generically
-        # for every service. Instead, the s3 config section entries are
-        # available as client context parameters. In the future, endpoint
-        # rulesets of services other than s3/s3control may require client
-        # context parameters.
-        client_context = (
-            s3_config_raw if self._is_s3_service(service_name_raw) else {}
-        )
+        # Client context params for s3 conflict with the available settings
+        # in the `s3` parameter on the `Config` object. If the same parameter
+        # is set in both places, the value in the `s3` parameter takes priority.
+        if client_config is not None:
+            client_context = client_config.client_context_params or {}
+        else:
+            client_context = {}
+        if self._is_s3_service(service_name_raw):
+            client_context.update(s3_config_raw)
+
         sig_version = (
             client_config.signature_version
             if client_config is not None

aiobotocore/client.py

@@ -378,7 +378,10 @@ async def _make_api_call(self, operation_name, api_params):
         )
 
         if http.status_code >= 300:
-            error_code = parsed_response.get("Error", {}).get("Code")
+            error_info = parsed_response.get("Error", {})
+            error_code = error_info.get("QueryErrorCode") or error_info.get(
+                "Code"
+            )
             error_class = self.exceptions.from_code(error_code)
             raise error_class(parsed_response, operation_name)
         else:
@@ -469,7 +472,7 @@ async def _resolve_endpoint_ruleset(
         returned.
 
         Use ignore_signing_region for generating presigned URLs or any other
-        situtation where the signing region information from the ruleset
+        situation where the signing region information from the ruleset
         resolver should be ignored.
 
         Returns tuple of URL and headers dictionary. Additionally, the

aiobotocore/credentials.py

@@ -82,6 +82,9 @@ def create_credential_resolver(session, cache=None, region_name=None):
             session
         ),
         'ec2_credential_refresh_window': _DEFAULT_ADVISORY_REFRESH_TIMEOUT,
+        'ec2_metadata_v1_disabled': session.get_config_variable(
+            'ec2_metadata_v1_disabled'
+        ),
     }
 
     if cache is None:

aiobotocore/utils.py

@@ -87,8 +87,10 @@ def __init__(
 
         if env is None:
             env = os.environ.copy()
-        self._disabled = env.get('AWS_EC2_METADATA_DISABLED', 'false').lower()
-        self._disabled = self._disabled == 'true'
+        self._disabled = (
+            env.get('AWS_EC2_METADATA_DISABLED', 'false').lower() == 'true'
+        )
+        self._imds_v1_disabled = config.get('ec2_metadata_v1_disabled')
         self._user_agent = user_agent
 
         self._session = session or _RefCountedSession(
@@ -144,6 +146,8 @@ async def _fetch_metadata_token(self):
 
     async def _get_request(self, url_path, retry_func, token=None):
         self._assert_enabled()
+        if not token:
+            self._assert_v1_enabled()
         if retry_func is None:
             retry_func = self._default_retry
         url = self._construct_url(url_path)
@@ -303,6 +307,9 @@ def _create_fetcher(self):
             'ec2_metadata_service_endpoint_mode': resolve_imds_endpoint_mode(
                 self._session
             ),
+            'ec2_metadata_v1_disabled': self._session.get_config_variable(
+                'ec2_metadata_v1_disabled'
+            ),
         }
         fetcher = AioInstanceMetadataRegionFetcher(
             timeout=metadata_timeout,
@@ -620,7 +627,7 @@ async def retrieve_full_uri(self, full_url, headers=None):
         return await self._retrieve_credentials(full_url, headers)
 
     async def retrieve_uri(self, relative_uri):
-        """Retrieve JSON metadata from ECS metadata.
+        """Retrieve JSON metadata from container metadata.
 
         :type relative_uri: str
         :param relative_uri: A relative URI, e.g "/foo/bar?id=123"
@@ -666,20 +673,20 @@ async def _get_response(self, full_url, headers, timeout):
                 if response.status_code != 200:
                     raise MetadataRetrievalError(
                         error_msg=(
-                            "Received non 200 response (%s) from ECS metadata: %s"
+                            f"Received non 200 response {response.status_code} "
+                            f"from container metadata: {response_text}"
                         )
-                        % (response.status_code, response_text)
                     )
                 try:
                     return json.loads(response_text)
                 except ValueError:
-                    error_msg = "Unable to parse JSON returned from ECS metadata services"
+                    error_msg = "Unable to parse JSON returned from container metadata services"
                     logger.debug('%s:%s', error_msg, response_text)
                     raise MetadataRetrievalError(error_msg=error_msg)
 
         except RETRYABLE_HTTP_ERRORS as e:
             error_msg = (
                 "Received error when attempting to retrieve "
-                "ECS metadata: %s" % e
+                f"container metadata: {e}"
             )
             raise MetadataRetrievalError(error_msg=error_msg)

requirements-dev.in

@@ -1,7 +1,7 @@
 codecov~=2.1.13
 coverage==7.2.7
 docker~=6.1.3
-moto[server,s3,sqs,lambda,dynamodb,cloudformation,sns,batch,ec2,rds]~=4.0.0
+moto[server,s3,sqs,lambda,dynamodb,cloudformation,sns,batch,ec2,rds]~=4.2.9
 pre-commit~=3.5.0
 pytest==7.4.0
 pytest-cov==4.1.0

setup.py

@@ -7,15 +7,15 @@
 # NOTE: When updating botocore make sure to update awscli/boto3 versions below
 install_requires = [
     # pegged to also match items in `extras_require`
-    'botocore>=1.31.16,<1.31.71',
+    'botocore>=1.32.4,<1.33.2',
     'aiohttp>=3.7.4.post0,<4.0.0',
     'wrapt>=1.10.10, <2.0.0',
     'aioitertools>=0.5.1,<1.0.0',
 ]
 
 extras_require = {
-    'awscli': ['awscli>=1.29.16,<1.29.71'],
-    'boto3': ['boto3>=1.28.16,<1.28.71'],
+    'awscli': ['awscli>=1.30.4,<1.31.2'],
+    'boto3': ['boto3>=1.29.4,<1.33.2'],
 }
 
 

tests/test_basic_s3.py

@@ -627,6 +627,7 @@ async def test_head_object_keys(s3_client, create_object, bucket_name):
     # this is to ensure things like:
     # https://github.com/aio-libs/aiobotocore/issues/131 don't happen again
     assert set(resp.keys()) == {
+        'AcceptRanges',
         'ETag',
         'ContentType',
         'Metadata',

tests/test_patches.py

@@ -137,7 +137,7 @@
         '2dc13a6f32c470bc415a2cfc1f82cf569b1a5196'
     },
     ClientArgsCreator._build_endpoint_resolver: {
-        '9aa226b8d6f09f7270633b8cc35bc82a15386ee4'
+        '0f80192233321ae4a55d95b68f5b8a68f3ad18e6',
     },
     # client.py
     ClientCreator.create_client: {'ef5bef8f4b2887143165e72554fd85c36af7e822'},
@@ -165,24 +165,24 @@
     ClientCreator._register_legacy_retries: {
         '000b2f2a122602e2e741ec2e89308dc2e2b67329'
     },
-    BaseClient._make_api_call: {'ea961baa7ea0b1a9b8318a3638b970e38ba4ac76'},
+    BaseClient._make_api_call: {
+        '1ac2e166cc8e5020224a808d2ccdfda18e6bdbf2',
+    },
     BaseClient._make_request: {'cfd8bbf19ea132134717cdf9c460694ddacdbf58'},
     BaseClient._convert_to_request_dict: {
         '5e0a374926b6ee1a8715963ab551e126506e7fc9'
     },
     BaseClient._emit_api_params: {'abd67874dae8d5cd2788412e4699398cb915a119'},
     BaseClient._resolve_endpoint_ruleset: {
-        'e8e7fe581a2e4ff1a75d1ee923c0ed2c6a0d9c9d'
+        '3206a73ae79601c42f8a5ae1d7e0e903a2495acb',
     },
     BaseClient.get_paginator: {'3531d5988aaaf0fbb3885044ccee1a693ec2608b'},
     BaseClient.get_waiter: {'44f0473d993d49ac7502984a7ccee3240b088404'},
     BaseClient.__getattr__: {'3ec17f468f50789fa633d6041f40b66a2f593e77'},
     # config.py
     Config.merge: {'c3dd8c3ffe0da86953ceba4a35267dfb79c6a2c8'},
     Config: {
-        '4153fcb2ddf68b86f3774da1016b9cbfa1659b0b',
-        'c6b76ca9e061c4fee99be96fb716a49043eb1806',
-        'ef03037bbe22945d5aa83bf39854e758f1b0c768',
+        '9154efda503c21ab9aa048214ddca7aa637e2ff9',
     },
     # credentials.py
     create_mfa_serial_refresher: {'9b5e98782fcacdcea5899a6d0d29d1b9de348bb0'},
@@ -317,7 +317,9 @@
     },
     BotoProvider.load: {'e84ebfe3d6698dc4683f0f6699d4a9907c87bebb'},
     OriginalEC2Provider.load: {'dc58cd1c79d177105b183a2d20e1154e6f8f0733'},
-    create_credential_resolver: {'f3501ad2330afe5e1ef4e71c7537f94885c73821'},
+    create_credential_resolver: {
+        'fe797afd33126be87f86e44ab20475d50d727a4e',
+    },
     get_credentials: {'ff0c735a388ac8dd7fe300a32c1e36cdf33c0f56'},
     # configprovider.py
     SmartDefaultsConfigStoreFactory.merge_smart_defaults: {
@@ -479,16 +481,20 @@
         '2c7080f7d6ee5a3dacc1b690945c045dba1b1d21'
     },
     ContainerMetadataFetcher.retrieve_uri: {
-        '4ee8aa704cf0a378d68ef9a7b375a1aa8840b000'
+        '9067ffe122e6fcff4a7f4dd2b7f5b3de5e1ea4ec',
     },
     ContainerMetadataFetcher._retrieve_credentials: {
         'b00694931af86ef1a9305ad29328030ee366cea9'
     },
     ContainerMetadataFetcher._get_response: {
-        '4dc84054db957f2c1fb2fb1b01eb462bd57b1a64'
+        'b2c2fe2d74ce1894168e8e052c4e97cc70539b1a',
+    },
+    IMDSFetcher.__init__: {
+        '50ea982d3b94d7301d39480f827cfc1502800cb4',
+    },
+    IMDSFetcher._get_request: {
+        '118354ef768da5a7402d5d2bf6f55b4fbb2525e4',
     },
-    IMDSFetcher.__init__: {'e7e62b79f6a9e4cb14120e61c4516f0a61148100'},
-    IMDSFetcher._get_request: {'7f8ad4724ac08300a0c55e307bfeb5abc0579d26'},
     IMDSFetcher._fetch_metadata_token: {
         '12225b35a73130632038785a8c2e6fbaaf9de1f4'
     },
@@ -541,7 +547,7 @@
         '4631ced79cff143de5d3fdf03cd69720778f141b'
     },
     IMDSRegionProvider._create_fetcher: {
-        '7031d7cdaea0244a5d860f2f8f6c013e25578123'
+        '18da52c786a20d91615258a8127b566688ecbb39',
     },
     # waiter.py
     NormalizedOperationMethod.__call__: {
@@ -559,7 +565,6 @@
     },
     # httpsession.py
     URLLib3Session: {
-        'c72094afb3aa62db0ade9be09be72ec7a2c3d80a',
         '1c418944abceb3a3d76c2c22348b4a39280d27ef',
     },
     EndpointDiscoveryHandler.discover_endpoint: {
@@ -609,7 +614,9 @@
     AWSResponse.content: {'307a4eb1d46360ef808a876d7d00cbbde6198eb1'},
     AWSResponse.text: {'a724100ba9f6d51b333b8fe470fac46376d5044a'},
     # httpchecksum.py
-    handle_checksum_body: {'4b9aeef18d816563624c66c57126d1ffa6fe1993'},
+    handle_checksum_body: {
+        '898cee7a7a5e5a02af7e0e65dcbb8122257b85df',
+    },
     _handle_bytes_response: {'0761c4590c6addbe8c674e40fca9f7dd375a184b'},
     AwsChunkedWrapper._make_chunk: {
         '097361692f0fd6c863a17dd695739629982ef7e4'
@@ -639,7 +646,7 @@
         '24b442126f0ff730be0ae64dc7158929d4d2fca7'
     },
     retryhandler.MaxAttemptsDecorator._should_retry: {
-        '8ba8bcec27974861ec35a7d91ce96db1c04833fe'
+        '581273f875bb779a9ff796df8c8597ec551abf97',
     },
     retryhandler.MultiChecker.__call__: {
         'e8302c52e1bbbb129b6f505633a4bc4ae1e5a34f'