use variable for client schemes, allowing override

This change is intended to make the default client implementations
more flexible so that their scheme can be customized. This can be
useful in scenarios where a subclass wants to implement a custom
scheme on e.g. a S3 compatible API [1] but with a custom scheme
so that the default S3 access is still also available.

[1] https://cloudpathlib.drivendata.org/stable/authentication/#accessing-custom-s3-compatible-object-stores

The tests have been updated to include a new s3-like rig which uses the
new scheme override functionality.

cloudpathlib/azure/azblobclient.py

@@ -37,6 +37,8 @@ class AzureBlobClient(Client):
     authentication options.
     """
 
+    cloud_prefix: str = "az://"
+
     def __init__(
         self,
         account_url: Optional[str] = None,
@@ -276,12 +278,14 @@ def _list_dir(
     ) -> Iterable[Tuple[AzureBlobPath, bool]]:
         if not cloud_path.container:
             for container in self.service_client.list_containers():
-                yield self.CloudPath(f"az://{container.name}"), True
+                yield self.CloudPath(f"{self.cloud_prefix}{container.name}"), True
 
                 if not recursive:
                     continue
 
-                yield from self._list_dir(self.CloudPath(f"az://{container.name}"), recursive=True)
+                yield from self._list_dir(
+                    self.CloudPath(f"{self.cloud_prefix}{container.name}"), recursive=True
+                )
             return
 
         container_client = self.service_client.get_container_client(cloud_path.container)
@@ -295,7 +299,9 @@ def _list_dir(
             paths = file_system_client.get_paths(path=cloud_path.blob, recursive=recursive)
 
             for path in paths:
-                yield self.CloudPath(f"az://{cloud_path.container}/{path.name}"), path.is_directory
+                yield self.CloudPath(
+                    f"{self.cloud_prefix}{cloud_path.container}/{path.name}"
+                ), path.is_directory
 
         else:
             if not recursive:
@@ -306,7 +312,9 @@ def _list_dir(
             for blob in blobs:
                 # walk_blobs returns folders with a trailing slash
                 blob_path = blob.name.rstrip("/")
-                blob_cloud_path = self.CloudPath(f"az://{cloud_path.container}/{blob_path}")
+                blob_cloud_path = self.CloudPath(
+                    f"{self.cloud_prefix}{cloud_path.container}/{blob_path}"
+                )
 
                 yield blob_cloud_path, (
                     isinstance(blob, BlobPrefix)

cloudpathlib/client.py

@@ -30,6 +30,8 @@ class Client(abc.ABC, Generic[BoundedCloudPath]):
     _cloud_meta: CloudImplementation
     _default_client = None
 
+    cloud_prefix: str
+
     def __init__(
         self,
         file_cache_mode: Optional[Union[str, FileCacheMode]] = None,

cloudpathlib/gs/gsclient.py

@@ -35,6 +35,8 @@ class GSClient(Client):
     options.
     """
 
+    cloud_prefix: str = "gs://"
+
     def __init__(
         self,
         application_credentials: Optional[Union[str, os.PathLike]] = None,
@@ -183,7 +185,8 @@ def _list_dir(self, cloud_path: GSPath, recursive=False) -> Iterable[Tuple[GSPat
                 )
 
             yield from (
-                (self.CloudPath(f"gs://{str(b)}"), True) for b in self.client.list_buckets()
+                (self.CloudPath(f"{self.cloud_prefix}{str(b)}"), True)
+                for b in self.client.list_buckets()
             )
             return
 
@@ -200,25 +203,30 @@ def _list_dir(self, cloud_path: GSPath, recursive=False) -> Iterable[Tuple[GSPat
                     # if we haven't surfaced this directory already
                     if parent not in yielded_dirs and str(parent) != ".":
                         yield (
-                            self.CloudPath(f"gs://{cloud_path.bucket}/{prefix}{parent}"),
+                            self.CloudPath(
+                                f"{self.cloud_prefix}{cloud_path.bucket}/{prefix}{parent}"
+                            ),
                             True,  # is a directory
                         )
                         yielded_dirs.add(parent)
-                yield (self.CloudPath(f"gs://{cloud_path.bucket}/{o.name}"), False)  # is a file
+                yield (
+                    self.CloudPath(f"{self.cloud_prefix}{cloud_path.bucket}/{o.name}"),
+                    False,
+                )  # is a file
         else:
             iterator = bucket.list_blobs(delimiter="/", prefix=prefix)
 
             # files must be iterated first for `.prefixes` to be populated:
             #   see: https://github.com/googleapis/python-storage/issues/863
             for file in iterator:
                 yield (
-                    self.CloudPath(f"gs://{cloud_path.bucket}/{file.name}"),
+                    self.CloudPath(f"{self.cloud_prefix}{cloud_path.bucket}/{file.name}"),
                     False,  # is a file
                 )
 
             for directory in iterator.prefixes:
                 yield (
-                    self.CloudPath(f"gs://{cloud_path.bucket}/{directory}"),
+                    self.CloudPath(f"{self.cloud_prefix}{cloud_path.bucket}/{directory}"),
                     True,  # is a directory
                 )
 

cloudpathlib/local/implementations/azure.py

@@ -16,6 +16,7 @@ class LocalAzureBlobClient(LocalClient):
     substitute when writing tests.
     """
 
+    cloud_prefix: str = "az://"
     _cloud_meta = local_azure_blob_implementation
 
     def __init__(self, *args, **kwargs):

cloudpathlib/local/implementations/gs.py

@@ -12,6 +12,7 @@ class LocalGSClient(LocalClient):
     substitute when writing tests.
     """
 
+    cloud_prefix: str = "gs://"
     _cloud_meta = local_gs_implementation
 
 

cloudpathlib/local/implementations/s3.py

@@ -12,6 +12,7 @@ class LocalS3Client(LocalClient):
     substitute when writing tests.
     """
 
+    cloud_prefix: str = "s3://"
     _cloud_meta = local_s3_implementation
 
 

cloudpathlib/s3/s3client.py

@@ -25,6 +25,8 @@ class S3Client(Client):
     instances. See documentation for the [`__init__` method][cloudpathlib.s3.s3client.S3Client.__init__]
     for detailed authentication options."""
 
+    cloud_prefix: str = "s3://"
+
     def __init__(
         self,
         aws_access_key_id: Optional[str] = None,
@@ -217,7 +219,7 @@ def _list_dir(self, cloud_path: S3Path, recursive=False) -> Iterable[Tuple[S3Pat
                 )
 
             yield from (
-                (self.CloudPath(f"s3://{b['Name']}"), True)
+                (self.CloudPath(f"{self.cloud_prefix}{b['Name']}"), True)
                 for b in self.client.list_buckets().get("Buckets", [])
             )
             return
@@ -241,7 +243,7 @@ def _list_dir(self, cloud_path: S3Path, recursive=False) -> Iterable[Tuple[S3Pat
                 canonical = result_prefix.get("Prefix").rstrip("/")  # keep a canonical form
                 if canonical not in yielded_dirs:
                     yield (
-                        self.CloudPath(f"s3://{cloud_path.bucket}/{canonical}"),
+                        self.CloudPath(f"{self.cloud_prefix}{cloud_path.bucket}/{canonical}"),
                         True,
                     )
                     yielded_dirs.add(canonical)
@@ -254,7 +256,9 @@ def _list_dir(self, cloud_path: S3Path, recursive=False) -> Iterable[Tuple[S3Pat
                     parent_canonical = prefix + str(parent).rstrip("/")
                     if parent_canonical not in yielded_dirs and str(parent) != ".":
                         yield (
-                            self.CloudPath(f"s3://{cloud_path.bucket}/{parent_canonical}"),
+                            self.CloudPath(
+                                f"{self.cloud_prefix}{cloud_path.bucket}/{parent_canonical}"
+                            ),
                             True,
                         )
                         yielded_dirs.add(parent_canonical)
@@ -267,15 +271,17 @@ def _list_dir(self, cloud_path: S3Path, recursive=False) -> Iterable[Tuple[S3Pat
                 # s3 fake directories have 0 size and end with "/"
                 if result_key.get("Key").endswith("/") and result_key.get("Size") == 0:
                     yield (
-                        self.CloudPath(f"s3://{cloud_path.bucket}/{canonical}"),
+                        self.CloudPath(f"{self.cloud_prefix}{cloud_path.bucket}/{canonical}"),
                         True,
                     )
                     yielded_dirs.add(canonical)
 
                 # yield object as file
                 else:
                     yield (
-                        self.CloudPath(f"s3://{cloud_path.bucket}/{result_key.get('Key')}"),
+                        self.CloudPath(
+                            f"{self.cloud_prefix}{cloud_path.bucket}/{result_key.get('Key')}"
+                        ),
                         False,
                     )
 

tests/conftest.py

@@ -17,7 +17,8 @@
 from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_fixed
 
 from cloudpathlib import AzureBlobClient, AzureBlobPath, GSClient, GSPath, S3Client, S3Path
-from cloudpathlib.cloudpath import implementation_registry
+from cloudpathlib.client import register_client_class
+from cloudpathlib.cloudpath import implementation_registry, register_path_class
 from cloudpathlib.local import (
     local_azure_blob_implementation,
     LocalAzureBlobClient,
@@ -307,8 +308,7 @@ def s3_rig(request, monkeypatch, assets_dir):
         bucket.objects.filter(Prefix=test_dir).delete()
 
 
-@fixture()
-def custom_s3_rig(request, monkeypatch, assets_dir):
+def _custom_s3_rig_helper(request, monkeypatch, assets_dir, path_class, client_class):
     """
     Custom S3 rig used to test the integrations with non-AWS S3-compatible object storages like
         - MinIO (https://min.io/)
@@ -370,8 +370,8 @@ def _spin_up_bucket():
         )
 
     rig = CloudProviderTestRig(
-        path_class=S3Path,
-        client_class=S3Client,
+        path_class=path_class,
+        client_class=client_class,
         drive=drive,
         test_dir=test_dir,
         live_server=live_server,
@@ -393,6 +393,43 @@ def _spin_up_bucket():
         bucket.objects.filter(Prefix=test_dir).delete()
 
 
+@fixture()
+def custom_s3_rig(request, monkeypatch, assets_dir):
+    """
+    Custom S3 rig used to test the integrations with non-AWS S3-compatible object storages like
+        - MinIO (https://min.io/)
+        - CEPH  (https://ceph.io/ceph-storage/object-storage/)
+        - others
+    """
+    yield from _custom_s3_rig_helper(request, monkeypatch, assets_dir, S3Path, S3Client)
+
+
+@register_path_class("mys3")
+class MyS3Path(S3Path):
+    cloud_prefix: str = "mys3://"
+
+
+@register_client_class("mys3")
+class MyS3Client(S3Client):
+    cloud_prefix: str = "mys3://"
+
+
+# Mirrors the definition of the S3Client class
+MyS3Client.MyS3Path = MyS3Client.CloudPath  # type: ignore
+
+
+@fixture()
+def custom_scheme_s3_rig(request, monkeypatch, assets_dir):
+    """
+    Custom S3 rig used to test the integrations with non-AWS S3-compatible object storages like
+        - MinIO (https://min.io/)
+        - CEPH  (https://ceph.io/ceph-storage/object-storage/)
+        - others
+    with the addition of a custom scheme being used.
+    """
+    yield from _custom_s3_rig_helper(request, monkeypatch, assets_dir, MyS3Path, MyS3Client)
+
+
 @fixture()
 def local_azure_rig(request, monkeypatch, assets_dir):
     drive = os.getenv("LIVE_AZURE_CONTAINER", DEFAULT_CONTAINER_NAME)
@@ -486,6 +523,7 @@ def local_s3_rig(request, monkeypatch, assets_dir):
         gs_rig,
         s3_rig,
         custom_s3_rig,
+        custom_scheme_s3_rig,
         local_azure_rig,
         local_s3_rig,
         local_gs_rig,
@@ -498,5 +536,6 @@ def local_s3_rig(request, monkeypatch, assets_dir):
     [
         s3_rig,
         custom_s3_rig,
+        custom_scheme_s3_rig,
     ],
 )