Merge branch 'main' into unpin-urllib3

.github/workflows/ci.yml

@@ -13,7 +13,7 @@ jobs:
       - name: Check out the code
         uses: actions/checkout@v3
 
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: "3.11"
 
@@ -23,7 +23,7 @@ jobs:
       - name: Determine dependencies
         run: poetry lock
 
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: "3.11"
           cache: poetry
@@ -40,7 +40,7 @@ jobs:
       - name: Check out the code
         uses: actions/checkout@v3
 
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: "3.11"
 
@@ -50,7 +50,7 @@ jobs:
       - name: Determine dependencies
         run: poetry lock
 
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: "3.11"
           cache: poetry
@@ -64,9 +64,9 @@ jobs:
   testing:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: "3.12"
 
@@ -76,7 +76,7 @@ jobs:
       - name: Determine dependencies
         run: poetry lock
 
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: "3.12"
           cache: poetry

.github/workflows/release-please.yml

@@ -22,9 +22,9 @@ jobs:
     needs: release-please
     if: ${{ needs.release-please.outputs.release_created }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: "3.11"
 
@@ -34,7 +34,7 @@ jobs:
       - name: Determine dependencies
         run: poetry lock
 
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: "3.11"
           cache: poetry

pyproject.toml

@@ -9,22 +9,20 @@ documentation = "https://snakemake.github.io/snakemake-plugin-catalog/plugins/st
 license = "MIT"
 keywords = ["snakemake", "plugin", "storage", "s3"]
 
-
 [tool.poetry.dependencies]
 python = "^3.11"
 snakemake-interface-common = "^1.14.0"
-snakemake-interface-storage-plugins = "^3.2.2"
+snakemake-interface-storage-plugins = "^3.2.4"
 # https://github.com/boto/botocore/commit/f9470df1af9f2a59d3c30f691e593d0e4abbe9e2
 boto3 = "^1.34.63"
 botocore = "^1.34.63"
 
-
 [tool.poetry.group.dev.dependencies]
 black = "^23.9.1"
 flake8 = "^6.1.0"
 coverage = "^7.3.1"
 pytest = "^7.4.2"
-snakemake = {git="https://github.com/snakemake/snakemake.git"}
+snakemake = "^8.18.0"
 setuptools = "*"
 
 [build-system]

snakemake_storage_plugin_s3/__init__.py

@@ -296,7 +296,8 @@ def retrieve_object(self):
         if self.is_dir():
             self.local_path().mkdir(parents=True, exist_ok=True)
             for item in self.get_subkeys():
-                subkey = item.key.split("/", 1)[1]
+                prefix_length = len(self.s3obj().key) + 1  # +1 for trailing slash
+                subkey = item.key[prefix_length:]
                 localpath = self.local_path() / subkey
                 localpath.parent.mkdir(parents=True, exist_ok=True)
                 item.Object().download_file(localpath)

tests/tests.py

@@ -8,7 +8,7 @@
 from snakemake.executors import local as local_executor
 from snakemake_storage_plugin_s3 import StorageProvider, StorageProviderSettings
 import snakemake.common.tests
-import snakemake.settings
+import snakemake.settings.types
 
 
 class TestStorageNoSettings(TestStorageBase):
@@ -17,7 +17,7 @@ class TestStorageNoSettings(TestStorageBase):
     files_only = False
 
     def get_query(self, tmp_path) -> str:
-        return "s3://snakemake-test-bucket/test-file.txt"
+        return "s3://snakemake-test-bucket/testdir1/testdir2/test-file.txt"
 
     def get_query_not_existing(self, tmp_path) -> str:
         bucket = uuid.uuid4().hex
@@ -58,5 +58,5 @@ def get_assume_shared_fs(self) -> bool:
 
     def get_remote_execution_settings(
         self,
-    ) -> snakemake.settings.RemoteExecutionSettings:
-        return snakemake.settings.RemoteExecutionSettings()
+    ) -> snakemake.settings.types.RemoteExecutionSettings:
+        return snakemake.settings.types.RemoteExecutionSettings()