setup local gcs test server

.github/workflows/ci.yml

@@ -84,6 +84,10 @@ jobs:
       - name: Install dependencies
         run: poetry install
 
+      - name: Run fake GCS server
+        run: |
+          docker run -d -p 5050:4443 -v storage_data:/storage fsouza/fake-gcs-server -scheme http
+
       - name: Run pytest
         run: poetry run coverage run -m pytest tests/tests.py
 

snakemake_storage_plugin_gs/__init__.py

@@ -23,6 +23,7 @@
 import google.cloud.exceptions
 from google.cloud import storage
 from google.api_core import retry
+from google.api_core.client_options import ClientOptions
 from google_crc32c import Checksum
 
 
@@ -67,6 +68,14 @@ class StorageProviderSettings(StorageProviderSettingsBase):
             "type": int,
         },
     )
+    api_endpoint: Optional[str] = field(
+        default=None,
+        metadata={
+            "help": "Google Cloud API endpoint",
+            "env_var": False,
+            "required": False,
+        },
+    )
 
 
 class Crc32cCalculator:
@@ -172,7 +181,7 @@ class StorageProvider(StorageProviderBase):
     # futher stuff.
 
     def __post_init__(self):
-        self.client = storage.Client()
+        self.client = storage.Client(client_options=ClientOptions(api_endpoint=self.settings.api_endpoint))
 
     @classmethod
     def is_valid_query(cls, query: str) -> StorageQueryValidationResult:
@@ -188,11 +197,11 @@ def is_valid_query(cls, query: str) -> StorageQueryValidationResult:
                 valid=False,
                 reason=f"cannot be parsed as URL ({e})",
             )
-        if parsed.scheme != "gs":
+        if parsed.scheme != "gcs":
             return StorageQueryValidationResult(
                 query=query,
                 valid=False,
-                reason="must start with gs (gs://...)",
+                reason="must start with gcs (gcs://...)",
             )
         return StorageQueryValidationResult(
             query=query,
@@ -206,8 +215,8 @@ def example_queries(cls) -> List[ExampleQuery]:
         """
         return [
             ExampleQuery(
-                query="gs://mybucket/myfile.txt",
-                description="A file in an google storage (GS) bucket",
+                query="gcs://mybucket/myfile.txt",
+                description="A file in an google storage (GCS) bucket",
             )
         ]
 
@@ -505,18 +514,6 @@ def client(self):
     def blob(self):
         return self.bucket.blob(self.key)
 
-    def parse(self):
-        """
-        TODO note from vsoch - this might belong in the provider parse or the
-        post init to validate the bucket / local file are correct.
-        """
-        m = re.search("(?P<bucket>[^/]*)/(?P<key>.*)", self.local_file())
-        if len(m.groups()) != 2:
-            raise WorkflowError(
-                "GS remote file {} does not have the form "
-                "<bucket>/<key>.".format(self.local_file())
-            )
-
     # Note from @vsoch - functions removed include:
     # name
     # list (seems to be on provider now)

tests/__init__.py

@@ -0,0 +1,33 @@
+from typing import List, Optional, Type
+import uuid
+from snakemake_interface_storage_plugins.tests import TestStorageBase
+from snakemake_interface_storage_plugins.storage_provider import StorageProviderBase
+from snakemake_interface_storage_plugins.settings import StorageProviderSettingsBase
+
+from snakemake_storage_plugin_s3 import StorageProvider, StorageProviderSettings
+
+
+class TestStorageNoSettings(TestStorageBase):
+    __test__ = True
+    retrieve_only = False
+
+    def get_query(self, tmp_path) -> str:
+        return "gcs://snakemake-test-bucket/test-file.txt"
+
+    def get_query_not_existing(self, tmp_path) -> str:
+        bucket = uuid.uuid4().hex
+        key = uuid.uuid4().hex
+        return f"gcs://{bucket}/{key}"
+
+    def get_storage_provider_cls(self) -> Type[StorageProviderBase]:
+        # Return the StorageProvider class of this plugin
+        return StorageProvider
+
+    def get_storage_provider_settings(self) -> Optional[StorageProviderSettingsBase]:
+        # instantiate StorageProviderSettings of this plugin as appropriate
+        return StorageProviderSettings(
+            api_endpoint="http://localhost:5050",
+        )
+
+    def get_example_args(self) -> List[str]:
+        return []