Skip to content

Commit

Permalink
fixes
Browse files Browse the repository at this point in the history
  • Loading branch information
johanneskoester committed Dec 7, 2023
1 parent fb48aaf commit 1a31dfc
Show file tree
Hide file tree
Showing 2 changed files with 17 additions and 33 deletions.
46 changes: 15 additions & 31 deletions snakemake_storage_plugin_gcs/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -338,47 +338,35 @@ def mtime(self) -> float:
"""
Return the modification time
"""
if self.exists():
if self.is_directory():
return max(
blob.updated.timestamp() for blob in self.directory_entries()
)
else:
self.update_blob()
return self.blob.updated.timestamp()
else:
raise WorkflowError(
"The file does not seem to exist remotely: %s" % self.local_file()
if self.is_directory():
return max(
blob.updated.timestamp() for blob in self.directory_entries()
)
else:
self.update_blob()
return self.blob.updated.timestamp()

@retry.Retry(predicate=google_cloud_retry_predicate)
def size(self) -> int:
"""
Return the size in bytes
"""
if self.exists():
if self.is_directory():
return 0
else:
self.update_blob()
return self.blob.size // 1024
if self.is_directory():
return 0
else:
return self._iofile.size_local
self.update_blob()
return self.blob.size // 1024

@retry.Retry(predicate=google_cloud_retry_predicate, deadline=600)
def retrieve_object(self):
"""
Ensure that the object is accessible locally under self.local_path()
In the previous GLS.py this was _download. We retry with 10 minutes.
"""
if not self.exists():
return None

# Create just a directory, or a file itself
if self.is_directory():
return self._download_directory()
return download_blob(self.blob, self.local_file())
self._download_directory()
else:
download_blob(self.blob, self.local_path())

# The following to methods are only required if the class inherits from
# StorageObjectReadWrite.
Expand All @@ -400,7 +388,7 @@ def store_object(self):
self.update_blob()

# Distinguish between single file, and folder
f = self.local_file()
f = self.local_path()
if os.path.isdir(f):
# Ensure the "directory" exists
self.blob.upload_from_string(
Expand Down Expand Up @@ -470,8 +458,7 @@ def _download_directory(self):
Handle download of a storage folder (assists retrieve_blob)
"""
# Create the directory locally
# TODO check that local_file is still valid
os.makedirs(self.local_file(), exist_ok=True)
self.local_path().mkdir(exist_ok=True)

for blob in self.directory_entries():
local_name = f"{blob.bucket.name}/{blob.name}"
Expand All @@ -482,9 +469,6 @@ def _download_directory(self):

download_blob(blob, local_name)

# Return the root directory
return self.local_file()

@retry.Retry(predicate=google_cloud_retry_predicate)
def update_blob(self):
"""
Expand Down
4 changes: 2 additions & 2 deletions tests/tests.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@

from snakemake_storage_plugin_gcs import StorageProvider, StorageProviderSettings

# Use local fake server as outlined here:
# https://github.com/fsouza/fake-gcs-server
os.environ["STORAGE_EMULATOR_HOST"] = "http://localhost:5050"


Expand All @@ -27,8 +29,6 @@ def get_storage_provider_cls(self) -> Type[StorageProviderBase]:

def get_storage_provider_settings(self) -> Optional[StorageProviderSettingsBase]:
# instantiate StorageProviderSettings of this plugin as appropriate
# Use local fake server as outlined here:
# https://www.claritician.com/how-to-mock-google-cloud-storage-during-development
return StorageProviderSettings()

def get_example_args(self) -> List[str]:
Expand Down

0 comments on commit 1a31dfc

Please sign in to comment.