use variable for client schemes, allowing override

cloudpathlib/azure/azblobclient.py

@@ -37,6 +37,8 @@
     authentication options.
     """
 
+    cloud_prefix: str = "az://"
+
     def __init__(
         self,
         account_url: Optional[str] = None,
@@ -276,12 +278,14 @@
     ) -> Iterable[Tuple[AzureBlobPath, bool]]:
         if not cloud_path.container:
             for container in self.service_client.list_containers():
-                yield self.CloudPath(f"az://{container.name}"), True
+                yield self.CloudPath(f"{self.cloud_prefix}{container.name}"), True
 
                 if not recursive:
                     continue
 
-                yield from self._list_dir(self.CloudPath(f"az://{container.name}"), recursive=True)
+                yield from self._list_dir(
+                    self.CloudPath(f"{self.cloud_prefix}{container.name}"), recursive=True
+                )
             return
 
         container_client = self.service_client.get_container_client(cloud_path.container)
@@ -295,7 +299,9 @@
             paths = file_system_client.get_paths(path=cloud_path.blob, recursive=recursive)
 
             for path in paths:
-                yield self.CloudPath(f"az://{cloud_path.container}/{path.name}"), path.is_directory
+                yield self.CloudPath(
+                    f"{self.cloud_prefix}{cloud_path.container}/{path.name}"
+                ), path.is_directory
 
         else:
             if not recursive:
@@ -306,7 +312,9 @@
             for blob in blobs:
                 # walk_blobs returns folders with a trailing slash
                 blob_path = blob.name.rstrip("/")
-                blob_cloud_path = self.CloudPath(f"az://{cloud_path.container}/{blob_path}")
+                blob_cloud_path = self.CloudPath(
+                    f"{self.cloud_prefix}{cloud_path.container}/{blob_path}"
+                )
 
                 yield blob_cloud_path, (
                     isinstance(blob, BlobPrefix)

cloudpathlib/client.py

@@ -30,6 +30,8 @@ class Client(abc.ABC, Generic[BoundedCloudPath]):
     _cloud_meta: CloudImplementation
     _default_client = None
 
+    cloud_prefix: str
+
     def __init__(
         self,
         file_cache_mode: Optional[Union[str, FileCacheMode]] = None,

cloudpathlib/gs/gsclient.py

@@ -35,6 +35,8 @@ class GSClient(Client):
     options.
     """
 
+    cloud_prefix: str = "gs://"
+
     def __init__(
         self,
         application_credentials: Optional[Union[str, os.PathLike]] = None,
@@ -183,7 +185,8 @@ def _list_dir(self, cloud_path: GSPath, recursive=False) -> Iterable[Tuple[GSPat
                 )
 
             yield from (
-                (self.CloudPath(f"gs://{str(b)}"), True) for b in self.client.list_buckets()
+                (self.CloudPath(f"{self.cloud_prefix}{str(b)}"), True)
+                for b in self.client.list_buckets()
             )
             return
 
@@ -200,25 +203,30 @@ def _list_dir(self, cloud_path: GSPath, recursive=False) -> Iterable[Tuple[GSPat
                     # if we haven't surfaced this directory already
                     if parent not in yielded_dirs and str(parent) != ".":
                         yield (
-                            self.CloudPath(f"gs://{cloud_path.bucket}/{prefix}{parent}"),
+                            self.CloudPath(
+                                f"{self.cloud_prefix}{cloud_path.bucket}/{prefix}{parent}"
+                            ),
                             True,  # is a directory
                         )
                         yielded_dirs.add(parent)
-                yield (self.CloudPath(f"gs://{cloud_path.bucket}/{o.name}"), False)  # is a file
+                yield (
+                    self.CloudPath(f"{self.cloud_prefix}{cloud_path.bucket}/{o.name}"),
+                    False,
+                )  # is a file
         else:
             iterator = bucket.list_blobs(delimiter="/", prefix=prefix)
 
             # files must be iterated first for `.prefixes` to be populated:
             #   see: https://github.com/googleapis/python-storage/issues/863
             for file in iterator:
                 yield (
-                    self.CloudPath(f"gs://{cloud_path.bucket}/{file.name}"),
+                    self.CloudPath(f"{self.cloud_prefix}{cloud_path.bucket}/{file.name}"),
                     False,  # is a file
                 )
 
             for directory in iterator.prefixes:
                 yield (
-                    self.CloudPath(f"gs://{cloud_path.bucket}/{directory}"),
+                    self.CloudPath(f"{self.cloud_prefix}{cloud_path.bucket}/{directory}"),
                     True,  # is a directory
                 )
 

cloudpathlib/local/implementations/azure.py

@@ -16,6 +16,7 @@ class LocalAzureBlobClient(LocalClient):
     substitute when writing tests.
     """
 
+    cloud_prefix: str = "az://"
     _cloud_meta = local_azure_blob_implementation
 
     def __init__(self, *args, **kwargs):

cloudpathlib/local/implementations/gs.py

@@ -12,6 +12,7 @@ class LocalGSClient(LocalClient):
     substitute when writing tests.
     """
 
+    cloud_prefix: str = "gs://"
     _cloud_meta = local_gs_implementation
 
 

cloudpathlib/local/implementations/s3.py

@@ -12,6 +12,7 @@ class LocalS3Client(LocalClient):
     substitute when writing tests.
     """
 
+    cloud_prefix: str = "s3://"
     _cloud_meta = local_s3_implementation
 
 

cloudpathlib/s3/s3client.py

@@ -25,6 +25,8 @@ class S3Client(Client):
     instances. See documentation for the [`__init__` method][cloudpathlib.s3.s3client.S3Client.__init__]
     for detailed authentication options."""
 
+    cloud_prefix: str = "s3://"
+
     def __init__(
         self,
         aws_access_key_id: Optional[str] = None,
@@ -217,7 +219,7 @@ def _list_dir(self, cloud_path: S3Path, recursive=False) -> Iterable[Tuple[S3Pat
                 )
 
             yield from (
-                (self.CloudPath(f"s3://{b['Name']}"), True)
+                (self.CloudPath(f"{self.cloud_prefix}{b['Name']}"), True)
                 for b in self.client.list_buckets().get("Buckets", [])
             )
             return
@@ -241,7 +243,7 @@ def _list_dir(self, cloud_path: S3Path, recursive=False) -> Iterable[Tuple[S3Pat
                 canonical = result_prefix.get("Prefix").rstrip("/")  # keep a canonical form
                 if canonical not in yielded_dirs:
                     yield (
-                        self.CloudPath(f"s3://{cloud_path.bucket}/{canonical}"),
+                        self.CloudPath(f"{self.cloud_prefix}{cloud_path.bucket}/{canonical}"),
                         True,
                     )
                     yielded_dirs.add(canonical)
@@ -254,7 +256,9 @@ def _list_dir(self, cloud_path: S3Path, recursive=False) -> Iterable[Tuple[S3Pat
                     parent_canonical = prefix + str(parent).rstrip("/")
                     if parent_canonical not in yielded_dirs and str(parent) != ".":
                         yield (
-                            self.CloudPath(f"s3://{cloud_path.bucket}/{parent_canonical}"),
+                            self.CloudPath(
+                                f"{self.cloud_prefix}{cloud_path.bucket}/{parent_canonical}"
+                            ),
                             True,
                         )
                         yielded_dirs.add(parent_canonical)
@@ -267,15 +271,17 @@ def _list_dir(self, cloud_path: S3Path, recursive=False) -> Iterable[Tuple[S3Pat
                 # s3 fake directories have 0 size and end with "/"
                 if result_key.get("Key").endswith("/") and result_key.get("Size") == 0:
                     yield (
-                        self.CloudPath(f"s3://{cloud_path.bucket}/{canonical}"),
+                        self.CloudPath(f"{self.cloud_prefix}{cloud_path.bucket}/{canonical}"),
                         True,
                     )
                     yielded_dirs.add(canonical)
 
                 # yield object as file
                 else:
                     yield (
-                        self.CloudPath(f"s3://{cloud_path.bucket}/{result_key.get('Key')}"),
+                        self.CloudPath(
+                            f"{self.cloud_prefix}{cloud_path.bucket}/{result_key.get('Key')}"
+                        ),
                         False,
                     )
 

tests/conftest.py

@@ -17,7 +17,8 @@
 from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_fixed
 
 from cloudpathlib import AzureBlobClient, AzureBlobPath, GSClient, GSPath, S3Client, S3Path
-from cloudpathlib.cloudpath import implementation_registry
+from cloudpathlib.client import register_client_class
+from cloudpathlib.cloudpath import implementation_registry, register_path_class
 from cloudpathlib.local import (
     local_azure_blob_implementation,
     LocalAzureBlobClient,
@@ -307,8 +308,7 @@ def s3_rig(request, monkeypatch, assets_dir):
         bucket.objects.filter(Prefix=test_dir).delete()
 
 
-@fixture()
-def custom_s3_rig(request, monkeypatch, assets_dir):
+def _custom_s3_rig_helper(request, monkeypatch, assets_dir, path_class, client_class):
     """
     Custom S3 rig used to test the integrations with non-AWS S3-compatible object storages like
         - MinIO (https://min.io/)
@@ -370,8 +370,8 @@ def _spin_up_bucket():
         )
 
     rig = CloudProviderTestRig(
-        path_class=S3Path,
-        client_class=S3Client,
+        path_class=path_class,
+        client_class=client_class,
         drive=drive,
         test_dir=test_dir,
         live_server=live_server,
@@ -393,6 +393,43 @@ def _spin_up_bucket():
         bucket.objects.filter(Prefix=test_dir).delete()
 
 
+@fixture()
+def custom_s3_rig(request, monkeypatch, assets_dir):
+    """
+    Custom S3 rig used to test the integrations with non-AWS S3-compatible object storages like
+        - MinIO (https://min.io/)
+        - CEPH  (https://ceph.io/ceph-storage/object-storage/)
+        - others
+    """
+    yield from _custom_s3_rig_helper(request, monkeypatch, assets_dir, S3Path, S3Client)
+
+
+@register_path_class("mys3")
+class MyS3Path(S3Path):
+    cloud_prefix: str = "mys3://"
+
+
+@register_client_class("mys3")
+class MyS3Client(S3Client):
+    cloud_prefix: str = "mys3://"
+
+
+# Mirrors the definition of the S3Client class
+MyS3Client.MyS3Path = MyS3Client.CloudPath  # type: ignore
+
+
+@fixture()
+def custom_scheme_s3_rig(request, monkeypatch, assets_dir):
+    """
+    Custom S3 rig used to test the integrations with non-AWS S3-compatible object storages like
+        - MinIO (https://min.io/)
+        - CEPH  (https://ceph.io/ceph-storage/object-storage/)
+        - others
+    with the addition of a custom scheme being used.
+    """
+    yield from _custom_s3_rig_helper(request, monkeypatch, assets_dir, MyS3Path, MyS3Client)
+
+
 @fixture()
 def local_azure_rig(request, monkeypatch, assets_dir):
     drive = os.getenv("LIVE_AZURE_CONTAINER", DEFAULT_CONTAINER_NAME)
@@ -486,6 +523,7 @@ def local_s3_rig(request, monkeypatch, assets_dir):
         gs_rig,
         s3_rig,
         custom_s3_rig,
+        custom_scheme_s3_rig,
         local_azure_rig,
         local_s3_rig,
         local_gs_rig,
@@ -498,5 +536,6 @@ def local_s3_rig(request, monkeypatch, assets_dir):
     [
         s3_rig,
         custom_s3_rig,
+        custom_scheme_s3_rig,
     ],
 )