diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 42b8085..9aec2c9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,7 +13,7 @@ jobs: - name: Check out the code uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: "3.11" @@ -23,7 +23,7 @@ jobs: - name: Determine dependencies run: poetry lock - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: "3.11" cache: poetry @@ -40,7 +40,7 @@ jobs: - name: Check out the code uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: "3.11" @@ -50,7 +50,7 @@ jobs: - name: Determine dependencies run: poetry lock - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: "3.11" cache: poetry @@ -64,9 +64,9 @@ jobs: testing: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: "3.12" @@ -76,7 +76,7 @@ jobs: - name: Determine dependencies run: poetry lock - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: "3.12" cache: poetry diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml index 414d0c8..1d4ee82 100644 --- a/.github/workflows/release-please.yml +++ b/.github/workflows/release-please.yml @@ -22,9 +22,9 @@ jobs: needs: release-please if: ${{ needs.release-please.outputs.release_created }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: "3.11" @@ -34,7 +34,7 @@ jobs: - name: Determine dependencies run: poetry lock - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: "3.11" cache: poetry diff --git a/pyproject.toml b/pyproject.toml index 8fedba4..4d6a041 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,22 +9,20 @@ documentation = "https://snakemake.github.io/snakemake-plugin-catalog/plugins/st license = "MIT" keywords = ["snakemake", "plugin", "storage", "s3"] - [tool.poetry.dependencies] python = "^3.11" snakemake-interface-common = "^1.14.0" -snakemake-interface-storage-plugins = "^3.2.2" +snakemake-interface-storage-plugins = "^3.2.4" # https://github.com/boto/botocore/commit/f9470df1af9f2a59d3c30f691e593d0e4abbe9e2 boto3 = "^1.34.63" botocore = "^1.34.63" - [tool.poetry.group.dev.dependencies] black = "^23.9.1" flake8 = "^6.1.0" coverage = "^7.3.1" pytest = "^7.4.2" -snakemake = {git="https://github.com/snakemake/snakemake.git"} +snakemake = "^8.18.0" setuptools = "*" [build-system] diff --git a/snakemake_storage_plugin_s3/__init__.py b/snakemake_storage_plugin_s3/__init__.py index 19ad966..ae47465 100644 --- a/snakemake_storage_plugin_s3/__init__.py +++ b/snakemake_storage_plugin_s3/__init__.py @@ -296,7 +296,8 @@ def retrieve_object(self): if self.is_dir(): self.local_path().mkdir(parents=True, exist_ok=True) for item in self.get_subkeys(): - subkey = item.key.split("/", 1)[1] + prefix_length = len(self.s3obj().key) + 1 # +1 for trailing slash + subkey = item.key[prefix_length:] localpath = self.local_path() / subkey localpath.parent.mkdir(parents=True, exist_ok=True) item.Object().download_file(localpath) diff --git a/tests/tests.py b/tests/tests.py index bbad0f4..ef975a1 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -8,7 +8,7 @@ from snakemake.executors import local as local_executor from snakemake_storage_plugin_s3 import StorageProvider, StorageProviderSettings import snakemake.common.tests -import snakemake.settings +import snakemake.settings.types class TestStorageNoSettings(TestStorageBase): @@ -17,7 +17,7 @@ class TestStorageNoSettings(TestStorageBase): files_only = False def get_query(self, tmp_path) -> str: - return "s3://snakemake-test-bucket/test-file.txt" + return "s3://snakemake-test-bucket/testdir1/testdir2/test-file.txt" def get_query_not_existing(self, tmp_path) -> str: bucket = uuid.uuid4().hex @@ -58,5 +58,5 @@ def get_assume_shared_fs(self) -> bool: def get_remote_execution_settings( self, - ) -> snakemake.settings.RemoteExecutionSettings: - return snakemake.settings.RemoteExecutionSettings() + ) -> snakemake.settings.types.RemoteExecutionSettings: + return snakemake.settings.types.RemoteExecutionSettings()