Allow bucket validation to be disabled in s3fs

Author: timj

python/lsst/resources/s3.py

@@ -25,13 +25,15 @@
 from typing import IO, TYPE_CHECKING, cast
 
 from botocore.exceptions import ClientError
+from botocore.handlers import validate_bucket_name
 from lsst.utils.timer import time_this
 
 from ._resourceHandles._baseResourceHandle import ResourceHandleProtocol
 from ._resourceHandles._s3ResourceHandle import S3ResourceHandle
 from ._resourcePath import ResourcePath
 from .s3utils import (
     _get_s3_connection_parameters,
+    _s3_validate_bucket,
     all_retryable_errors,
     backoff,
     bucketExists,
@@ -335,6 +337,12 @@ def to_fsspec(self) -> tuple[AbstractFileSystem, str]:
             key=endpoint_config.access_key_id,
             secret=endpoint_config.secret_access_key,
         )
+        if not _s3_validate_bucket():
+            # Force the boto client to be constructed so we can modify it.
+            with contextlib.suppress(FileNotFoundError):
+                s3.open("bad_bucket/path")
+            s3.s3.meta.events.unregister("before-parameter-build.s3", validate_bucket_name)
+
         return s3, f"{self._bucket}/{self.relativeToPathRoot}"
 
     def _as_local(self) -> tuple[str, bool]:

python/lsst/resources/s3utils.py

@@ -221,14 +221,24 @@ def getS3Client(profile: str | None = None) -> boto3.client:
     if botocore is None:
         raise ModuleNotFoundError("Could not find botocore. Are you sure it is installed?")
 
-    disable_value = os.environ.get("LSST_DISABLE_BUCKET_VALIDATION", "0")
-    skip_validation = not re.search(r"^(0|f|n|false)?$", disable_value, re.I)
-
+    skip_validation = not _s3_validate_bucket()
     endpoint_config = _get_s3_connection_parameters(profile)
 
     return _get_s3_client(endpoint_config, skip_validation)
 
 
+def _s3_validate_bucket() -> bool:
+    """Indicate whether bucket validation should be enabled.
+
+    Returns
+    -------
+    validate : `bool`
+        If `True` bucket names should be validated.
+    """
+    disable_value = os.environ.get("LSST_DISABLE_BUCKET_VALIDATION", "0")
+    return bool(re.search(r"^(0|f|n|false)?$", disable_value, re.I))
+
+
 def _get_s3_connection_parameters(profile: str | None = None) -> _EndpointConfig:
     """Calculate the connection details.