Allow bucket validation to be disabled in s3fs

Author: timj

python/lsst/resources/s3.py

@@ -32,6 +32,8 @@
 from ._resourcePath import ResourcePath
 from .s3utils import (
     _get_s3_connection_parameters,
+    _s3_disable_bucket_validation,
+    _s3_should_validate_bucket,
     all_retryable_errors,
     backoff,
     bucketExists,
@@ -335,6 +337,11 @@ def to_fsspec(self) -> tuple[AbstractFileSystem, str]:
             key=endpoint_config.access_key_id,
             secret=endpoint_config.secret_access_key,
         )
+        if not _s3_should_validate_bucket():
+            # Accessing the s3 property forces the boto client to be
+            # constructed and cached and allows the validation to be removed.
+            _s3_disable_bucket_validation(s3.s3)
+
         return s3, f"{self._bucket}/{self.relativeToPathRoot}"
 
     def _as_local(self) -> tuple[str, bool]:

python/lsst/resources/s3utils.py

@@ -221,12 +221,21 @@ def getS3Client(profile: str | None = None) -> boto3.client:
     if botocore is None:
         raise ModuleNotFoundError("Could not find botocore. Are you sure it is installed?")
 
-    disable_value = os.environ.get("LSST_DISABLE_BUCKET_VALIDATION", "0")
-    skip_validation = not re.search(r"^(0|f|n|false)?$", disable_value, re.I)
-
     endpoint_config = _get_s3_connection_parameters(profile)
 
-    return _get_s3_client(endpoint_config, skip_validation)
+    return _get_s3_client(endpoint_config, not _s3_should_validate_bucket())
+
+
+def _s3_should_validate_bucket() -> bool:
+    """Indicate whether bucket validation should be enabled.
+
+    Returns
+    -------
+    validate : `bool`
+        If `True` bucket names should be validated.
+    """
+    disable_value = os.environ.get("LSST_DISABLE_BUCKET_VALIDATION", "0")
+    return bool(re.search(r"^(0|f|n|false)?$", disable_value, re.I))
 
 
 def _get_s3_connection_parameters(profile: str | None = None) -> _EndpointConfig:
@@ -254,6 +263,20 @@ def _get_s3_connection_parameters(profile: str | None = None) -> _EndpointConfig
     return _parse_endpoint_config(endpoint, profile)
 
 
+def _s3_disable_bucket_validation(client: boto3.client) -> None:
+    """Disable the bucket name validation in the client.
+
+    This removes the ``validate_bucket_name`` handler from the handlers
+    registered for this client.
+
+    Parameters
+    ----------
+    client : `boto3.client`
+        The client to modify.
+    """
+    client.meta.events.unregister("before-parameter-build.s3", validate_bucket_name)
+
+
 @functools.lru_cache
 def _get_s3_client(endpoint_config: _EndpointConfig, skip_validation: bool) -> boto3.client:
     # Helper function to cache the client for this endpoint
@@ -269,7 +292,7 @@ def _get_s3_client(endpoint_config: _EndpointConfig, skip_validation: bool) -> b
         config=config,
     )
     if skip_validation:
-        client.meta.events.unregister("before-parameter-build.s3", validate_bucket_name)
+        _s3_disable_bucket_validation(client)
     return client