Merge branch 'main' into session_refresh

fsspec · Feb 18, 2025 · d6aea66 · d6aea66
2 parents e134659 + 744d9c6
commit d6aea66
Show file tree

Hide file tree

Showing 4 changed files with 29 additions and 15 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -44,7 +44,7 @@ jobs:
 
       - name: Run Tests
         shell: bash -l {0}
-        run: pytest -vv -s s3fs -k test_write_blocks
+        run: pytest -vv -s s3fs
 
 
   pre-commit:

diff --git a/s3fs/tests/derived/s3fs_test.py b/s3fs/tests/derived/s3fs_test.py
@@ -16,8 +16,20 @@ class TestS3fsPut(abstract.AbstractPutTests, S3fsFixtures):
     pass
 
 
+def botocore_too_old():
+    import botocore
+    from packaging.version import parse
+
+    MIN_BOTOCORE_VERSION = "1.33.2"
+
+    return parse(botocore.__version__) < parse(MIN_BOTOCORE_VERSION)
+
+
 class TestS3fsPipe(abstract.AbstractPipeTests, S3fsFixtures):
-    pass
+
+    test_pipe_exclusive = pytest.mark.skipif(
+        botocore_too_old(), reason="Older botocore doesn't support exclusive writes"
+    )(abstract.AbstractPipeTests.test_pipe_exclusive)
 
 
 class TestS3fsOpen(abstract.AbstractOpenTests, S3fsFixtures):

diff --git a/s3fs/tests/test_s3fs.py b/s3fs/tests/test_s3fs.py
@@ -272,7 +272,7 @@ def test_info(s3):
     info.pop("ContentType")
     linfo.pop("Key")
     linfo.pop("Size")
-    linfo.pop("ChecksumAlgorithm")
+    linfo.pop("ChecksumAlgorithm", None)  # field DNE in some S3-compatible providers
     assert info == linfo
     parent = a.rsplit("/", 1)[0]
     s3.invalidate_cache()  # remove full path from the cache

diff --git a/s3fs/utils.py b/s3fs/utils.py
@@ -38,19 +38,21 @@ async def get_bucket_client(self, bucket_name=None):
         try:
             response = await general_client.head_bucket(Bucket=bucket_name)
         except ClientError as e:
-            region = (
-                e.response["ResponseMetadata"]
-                .get("HTTPHeaders", {})
-                .get("x-amz-bucket-region")
+            logger.debug("RC: HEAD_BUCKET call for %r has failed", bucket_name)
+            response = e.response
+
+        region = (
+            response["ResponseMetadata"]
+            .get("HTTPHeaders", {})
+            .get("x-amz-bucket-region")
+        )
+
+        if not region:
+            logger.debug(
+                "RC: No region in HEAD_BUCKET call response for %r, returning the general client",
+                bucket_name,
             )
-            if not region:
-                logger.debug(
-                    "RC: HEAD_BUCKET call for %r has failed, returning the general client",
-                    bucket_name,
-                )
-                return general_client
-        else:
-            region = response["ResponseMetadata"]["HTTPHeaders"]["x-amz-bucket-region"]
+            return general_client
 
         if region not in self._regions:
             logger.debug(