From 7ae855252700cfe8a55beede3291c0499edbe750 Mon Sep 17 00:00:00 2001 From: Peter Bull Date: Wed, 14 Aug 2024 15:52:42 -0700 Subject: [PATCH 01/14] Add pytest and loosen versions --- .gitignore | 1 + Makefile | 3 +++ pyproject.toml | 2 +- requirements-dev.txt | 6 ++++-- 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 59c8c813..8363e9b8 100644 --- a/.gitignore +++ b/.gitignore @@ -59,6 +59,7 @@ coverage.xml *.cover .hypothesis/ .pytest_cache/ +reportlog.jsonl # Translations *.mo diff --git a/Makefile b/Makefile index 51831642..9464111b 100644 --- a/Makefile +++ b/Makefile @@ -86,5 +86,8 @@ test-debug: ## rerun tests that failed in last run and stop with pdb at failure test-live-cloud: ## run tests on live cloud backends USE_LIVE_CLOUD=1 python -m pytest -vv +test-time-report: + pytest-duration-insights explore --no-trim reportlog.jsonl + perf: ## run performance measurement suite for s3 and save results to perf-results.csv python tests/performance/cli.py s3 --save-csv=perf-results.csv diff --git a/pyproject.toml b/pyproject.toml index c7f6dcdb..c7cf27d1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,7 +65,7 @@ ignore_missing_imports = true [tool.pytest.ini_options] minversion = "6.0" -addopts = "--cov=cloudpathlib --cov-report=term --cov-report=html --cov-report=xml -n=auto" +addopts = "--cov=cloudpathlib --cov-report=term --cov-report=html --cov-report=xml -n=auto --report-log reportlog.jsonl" testpaths = ["tests"] [tool.coverage.run] diff --git a/requirements-dev.txt b/requirements-dev.txt index fcb8eed3..7c526692 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -19,9 +19,11 @@ pandas pillow psutil pydantic -pytest<8 -pytest-cases>=3.7.0 +pytest +pytest-cases pytest-cov +pytest-duration-insights +pytest-reportlog pytest-xdist python-dotenv pywin32; sys_platform == 'win32' From d15abd0ee8829235fb2a31b6beb2a8ad5274dacc Mon Sep 17 00:00:00 2001 From: Peter Bull Date: Wed, 14 Aug 2024 17:18:49 -0700 Subject: [PATCH 02/14] Cache strategy --- tests/test_caching.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/test_caching.py b/tests/test_caching.py index 71890c43..b586c80d 100644 --- a/tests/test_caching.py +++ b/tests/test_caching.py @@ -4,7 +4,12 @@ from pathlib import Path import pytest -from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_random_exponential +from tenacity import ( + retry, + retry_if_exception_type, + stop_after_attempt, + wait_random_exponential, +) from cloudpathlib.enums import FileCacheMode from cloudpathlib.exceptions import ( From 7f37c37a470d371e10c3769b024f7334bdaec6ee Mon Sep 17 00:00:00 2001 From: Peter Bull Date: Wed, 14 Aug 2024 22:15:01 -0700 Subject: [PATCH 03/14] debug code --- tests/test_caching.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/tests/test_caching.py b/tests/test_caching.py index b586c80d..17bde966 100644 --- a/tests/test_caching.py +++ b/tests/test_caching.py @@ -1,5 +1,6 @@ import gc import os +import sys from time import sleep from pathlib import Path @@ -501,12 +502,37 @@ def test_manual_cache_clearing(rig: CloudProviderTestRig): assert len(list(filter(lambda x: x.is_file(), client._local_cache_dir.rglob("*")))) == 0 + # Enable debugging for garbage collection + gc.callbacks.append(lambda event, args: print(f"GC {event} - {args}")) + # also removes containing folder on client cleanted up local_cache_path = cp._local client_cache_folder = client._local_cache_dir del cp del client + def _debug(path): + import subprocess + import psutil + + # file handles on windows + if sys.platform == "win32": + import subprocess + + result = subprocess.run(["handle.exe", path], capture_output=True, text=True) + print(f" HANDLES FOR {path}") + print(result.stdout) + + # processes with open files + open_files = [] + for proc in psutil.process_iter(["pid", "name", "open_files"]): + for file in proc.info["open_files"] or []: + if path in file.path: + open_files.append((proc.info["pid"], proc.info["name"], file.path)) + + print(f" OPEN FILES INFO FOR {path}") + print(open_files) + # in CI there can be a lag before the cleanup actually happens @retry( retry=retry_if_exception_type(AssertionError), @@ -515,6 +541,9 @@ def test_manual_cache_clearing(rig: CloudProviderTestRig): reraise=True, ) def _resilient_assert(): + _debug(str(local_cache_path.resolve())) + _debug(str(client_cache_folder.resolve())) + gc.collect() # force gc before asserting assert not local_cache_path.exists() @@ -522,6 +551,8 @@ def _resilient_assert(): _resilient_assert() + gc.callbacks.pop() + def test_reuse_cache_after_manual_cache_clear(rig: CloudProviderTestRig): # use client that we can delete rather than default From 82ba89140ee26e5d9cc35684ceecbe95aac58193 Mon Sep 17 00:00:00 2001 From: Peter Bull Date: Wed, 14 Aug 2024 22:17:42 -0700 Subject: [PATCH 04/14] remove extra import --- tests/test_caching.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_caching.py b/tests/test_caching.py index 17bde966..bdea1475 100644 --- a/tests/test_caching.py +++ b/tests/test_caching.py @@ -512,7 +512,6 @@ def test_manual_cache_clearing(rig: CloudProviderTestRig): del client def _debug(path): - import subprocess import psutil # file handles on windows From dedf5d453834422b56361e0f94fd84c844047e86 Mon Sep 17 00:00:00 2001 From: Peter Bull Date: Wed, 14 Aug 2024 22:35:37 -0700 Subject: [PATCH 05/14] no handle.exe on gha --- tests/test_caching.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/tests/test_caching.py b/tests/test_caching.py index bdea1475..3360737e 100644 --- a/tests/test_caching.py +++ b/tests/test_caching.py @@ -1,6 +1,5 @@ import gc import os -import sys from time import sleep from pathlib import Path @@ -514,14 +513,6 @@ def test_manual_cache_clearing(rig: CloudProviderTestRig): def _debug(path): import psutil - # file handles on windows - if sys.platform == "win32": - import subprocess - - result = subprocess.run(["handle.exe", path], capture_output=True, text=True) - print(f" HANDLES FOR {path}") - print(result.stdout) - # processes with open files open_files = [] for proc in psutil.process_iter(["pid", "name", "open_files"]): From b9344b6f605dded01cc615169d289429660d62cd Mon Sep 17 00:00:00 2001 From: Peter Bull Date: Tue, 20 Aug 2024 16:47:04 -0700 Subject: [PATCH 06/14] resilient cache tests --- tests/test_caching.py | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/tests/test_caching.py b/tests/test_caching.py index 3360737e..47e09a59 100644 --- a/tests/test_caching.py +++ b/tests/test_caching.py @@ -438,13 +438,30 @@ def test_environment_variables_force_overwrite_to(rig: CloudProviderTestRig, tmp assert p.stat().st_mtime >= new_local.stat().st_mtime # would raise if not set - sleep(1.01) # give time so not equal when rounded - p._upload_file_to_cloud(new_local) - assert p.stat().st_mtime > orig_cloud_mod_time # cloud now overwritten + @retry( + retry=retry_if_exception_type(AssertionError), + wait=wait_random_exponential(multiplier=0.5, max=5), + stop=stop_after_attempt(10), + reraise=True, + ) + def _wait_for_cloud_newer(): + p._upload_file_to_cloud(new_local) + assert p.stat().st_mtime > orig_cloud_mod_time # cloud now overwritten + + _wait_for_cloud_newer() new_also_cloud = rig.create_cloud_path("dir_0/another_cloud_file.txt") - sleep(1.01) # give time so not equal when rounded - new_also_cloud.write_text("newer") + + @retry( + retry=retry_if_exception_type(OverwriteNewerLocalError), + wait=wait_random_exponential(multiplier=0.5, max=5), + stop=stop_after_attempt(10), + reraise=True, + ) + def _retry_write_until_old_enough(): + new_also_cloud.write_text("newer") + + _retry_write_until_old_enough() new_cloud_mod_time = new_also_cloud.stat().st_mtime From c579ec09175dde6fc1dafecdcfd0d2930955cef4 Mon Sep 17 00:00:00 2001 From: Peter Bull Date: Tue, 20 Aug 2024 16:52:15 -0700 Subject: [PATCH 07/14] initial sleep --- tests/test_caching.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_caching.py b/tests/test_caching.py index 47e09a59..85ec3ee3 100644 --- a/tests/test_caching.py +++ b/tests/test_caching.py @@ -452,6 +452,8 @@ def _wait_for_cloud_newer(): new_also_cloud = rig.create_cloud_path("dir_0/another_cloud_file.txt") + sleep(0.1) # at least a little different + @retry( retry=retry_if_exception_type(OverwriteNewerLocalError), wait=wait_random_exponential(multiplier=0.5, max=5), From 197a058f59d9791a0cc93e57b24679b4bf86434a Mon Sep 17 00:00:00 2001 From: Peter Bull Date: Thu, 22 Aug 2024 10:26:22 -0400 Subject: [PATCH 08/14] weakref finalize instead of __del__ --- cloudpathlib/cache_utils.py | 10 ++++++++++ cloudpathlib/client.py | 27 +++++++++++++++------------ cloudpathlib/cloudpath.py | 29 +++++++++++++++++++++++------ 3 files changed, 48 insertions(+), 18 deletions(-) create mode 100644 cloudpathlib/cache_utils.py diff --git a/cloudpathlib/cache_utils.py b/cloudpathlib/cache_utils.py new file mode 100644 index 00000000..ebf74ebc --- /dev/null +++ b/cloudpathlib/cache_utils.py @@ -0,0 +1,10 @@ +from pathlib import Path +import shutil + + +def _clear_cache(_local: Path) -> None: + if _local.exists(): + if _local.is_file(): + _local.unlink() + else: + shutil.rmtree(_local) diff --git a/cloudpathlib/client.py b/cloudpathlib/client.py index 1b6c32eb..f8ff595a 100644 --- a/cloudpathlib/client.py +++ b/cloudpathlib/client.py @@ -5,7 +5,9 @@ import shutil from tempfile import TemporaryDirectory from typing import Generic, Callable, Iterable, Optional, Tuple, TypeVar, Union +import weakref +from .cache_utils import _clear_cache from .cloudpath import CloudImplementation, CloudPath, implementation_registry from .enums import FileCacheMode from .exceptions import InvalidConfigurationException @@ -26,6 +28,18 @@ def decorator(cls: type) -> type: return decorator +def _client_finalizer(file_cache_mode: FileCacheMode, _local_cache_dir) -> None: + if file_cache_mode in [ + FileCacheMode.tmp_dir, + FileCacheMode.close_file, + FileCacheMode.cloudpath_object, + ]: + _clear_cache(_local_cache_dir) + + if _local_cache_dir.exists(): + _local_cache_dir.rmdir() + + class Client(abc.ABC, Generic[BoundedCloudPath]): _cloud_meta: CloudImplementation _default_client = None @@ -82,18 +96,7 @@ def __init__( self.file_cache_mode = file_cache_mode - def __del__(self) -> None: - # remove containing dir, even if a more aggressive strategy - # removed the actual files - if getattr(self, "file_cache_mode", None) in [ - FileCacheMode.tmp_dir, - FileCacheMode.close_file, - FileCacheMode.cloudpath_object, - ]: - self.clear_cache() - - if self._local_cache_dir.exists(): - self._local_cache_dir.rmdir() + weakref.finalize(self, _client_finalizer, self.file_cache_mode, self._local_cache_dir) @classmethod def get_default_client(cls) -> "Client": diff --git a/cloudpathlib/cloudpath.py b/cloudpathlib/cloudpath.py index d7bf391b..768b46d3 100644 --- a/cloudpathlib/cloudpath.py +++ b/cloudpathlib/cloudpath.py @@ -11,7 +11,6 @@ _PathParents, ) -import shutil import sys from typing import ( overload, @@ -33,6 +32,7 @@ ) from urllib.parse import urlparse from warnings import warn +import weakref if sys.version_info >= (3, 10): from typing import TypeGuard @@ -58,6 +58,7 @@ def _make_selector(pattern_parts, _flavour, case_sensitive=True): from . import anypath +from .cache_utils import _clear_cache from .exceptions import ( ClientMismatchError, CloudPathFileExistsError, @@ -188,6 +189,20 @@ def __init__(cls, name: str, bases: Tuple[type, ...], dic: Dict[str, Any]) -> No getattr(cls, attr).fget.__doc__ = docstring +def _cloudpath_finalizer(handle: Optional[IO], client: Optional["Client"], _no_prefix: str): + """Use weakref.finalizer instead of __del__ since it is more reliable (does + not wait for garbage collection to actually run). + """ + # make sure that file handle to local path is closed + if handle is not None: + handle.close() + + # ensure file removed from cache when cloudpath object deleted + if client is not None: + if getattr(client, "file_cache_mode", None) == FileCacheMode.cloudpath_object: + _clear_cache(client._local_cache_dir / _no_prefix) + + # Abstract base class class CloudPath(metaclass=CloudPathMeta): """Base class for cloud storage file URIs, in the style of the Python standard library's @@ -242,6 +257,9 @@ def __init__( # track if local has been written to, if so it may need to be uploaded self._dirty = False + # register cache cleanup method when this object is marked for garbage collection + weakref.finalize(self, _cloudpath_finalizer, self._handle, self._client, self._no_prefix) + @property def client(self): if getattr(self, "_client", None) is None: @@ -249,6 +267,7 @@ def client(self): return self._client +<<<<<<< HEAD def __del__(self) -> None: # make sure that file handle to local path is closed if self._handle is not None and self._local.exists(): @@ -259,6 +278,8 @@ def __del__(self) -> None: if getattr(client, "file_cache_mode", None) == FileCacheMode.cloudpath_object: self.clear_cache() +======= +>>>>>>> e33b583 (weakref finalize instead of __del__) def __getstate__(self) -> Dict[str, Any]: state = self.__dict__.copy() @@ -1088,11 +1109,7 @@ def copytree(self, destination, force_overwrite_to_cloud=None, ignore=None): def clear_cache(self): """Removes cache if it exists""" - if self._local.exists(): - if self._local.is_file(): - self._local.unlink() - else: - shutil.rmtree(self._local) + _clear_cache(self._local) # =========== private cloud methods =============== @property From de5c0b6313ad6a167d3078f0fb6d9700677e1b52 Mon Sep 17 00:00:00 2001 From: Peter Bull Date: Thu, 22 Aug 2024 11:16:12 -0400 Subject: [PATCH 09/14] delay on write check --- tests/test_caching.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test_caching.py b/tests/test_caching.py index 85ec3ee3..f868f1b3 100644 --- a/tests/test_caching.py +++ b/tests/test_caching.py @@ -455,19 +455,19 @@ def _wait_for_cloud_newer(): sleep(0.1) # at least a little different @retry( - retry=retry_if_exception_type(OverwriteNewerLocalError), + retry=retry_if_exception_type((OverwriteNewerLocalError, AssertionError)), wait=wait_random_exponential(multiplier=0.5, max=5), stop=stop_after_attempt(10), reraise=True, ) def _retry_write_until_old_enough(): new_also_cloud.write_text("newer") + new_cloud_mod_time = new_also_cloud.stat().st_mtime + assert p.stat().st_mtime < new_cloud_mod_time # would raise if not set + return new_cloud_mod_time - _retry_write_until_old_enough() + new_cloud_mod_time = _retry_write_until_old_enough() - new_cloud_mod_time = new_also_cloud.stat().st_mtime - - assert p.stat().st_mtime < new_cloud_mod_time # would raise if not set p.copy(new_also_cloud) assert new_also_cloud.stat().st_mtime >= new_cloud_mod_time From 8ab474529b867fd207530fcf7bbb777e54559ad3 Mon Sep 17 00:00:00 2001 From: Peter Bull Date: Thu, 22 Aug 2024 12:26:44 -0400 Subject: [PATCH 10/14] remove debug code --- tests/test_caching.py | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/tests/test_caching.py b/tests/test_caching.py index f868f1b3..a5cb67b7 100644 --- a/tests/test_caching.py +++ b/tests/test_caching.py @@ -520,28 +520,12 @@ def test_manual_cache_clearing(rig: CloudProviderTestRig): assert len(list(filter(lambda x: x.is_file(), client._local_cache_dir.rglob("*")))) == 0 - # Enable debugging for garbage collection - gc.callbacks.append(lambda event, args: print(f"GC {event} - {args}")) - # also removes containing folder on client cleanted up local_cache_path = cp._local client_cache_folder = client._local_cache_dir del cp del client - def _debug(path): - import psutil - - # processes with open files - open_files = [] - for proc in psutil.process_iter(["pid", "name", "open_files"]): - for file in proc.info["open_files"] or []: - if path in file.path: - open_files.append((proc.info["pid"], proc.info["name"], file.path)) - - print(f" OPEN FILES INFO FOR {path}") - print(open_files) - # in CI there can be a lag before the cleanup actually happens @retry( retry=retry_if_exception_type(AssertionError), @@ -550,9 +534,6 @@ def _debug(path): reraise=True, ) def _resilient_assert(): - _debug(str(local_cache_path.resolve())) - _debug(str(client_cache_folder.resolve())) - gc.collect() # force gc before asserting assert not local_cache_path.exists() @@ -560,8 +541,6 @@ def _resilient_assert(): _resilient_assert() - gc.callbacks.pop() - def test_reuse_cache_after_manual_cache_clear(rig: CloudProviderTestRig): # use client that we can delete rather than default From dc04dff0d6fd09ca4467e34f4c18aa35367ad2cd Mon Sep 17 00:00:00 2001 From: Peter Bull Date: Wed, 28 Aug 2024 13:49:10 -0700 Subject: [PATCH 11/14] remove merge cruft --- cloudpathlib/cloudpath.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/cloudpathlib/cloudpath.py b/cloudpathlib/cloudpath.py index 768b46d3..e2335127 100644 --- a/cloudpathlib/cloudpath.py +++ b/cloudpathlib/cloudpath.py @@ -267,19 +267,6 @@ def client(self): return self._client -<<<<<<< HEAD - def __del__(self) -> None: - # make sure that file handle to local path is closed - if self._handle is not None and self._local.exists(): - self._handle.close() - - # ensure file removed from cache when cloudpath object deleted - client = getattr(self, "_client", None) - if getattr(client, "file_cache_mode", None) == FileCacheMode.cloudpath_object: - self.clear_cache() - -======= ->>>>>>> e33b583 (weakref finalize instead of __del__) def __getstate__(self) -> Dict[str, Any]: state = self.__dict__.copy() From f395a5aa927e52c668430e7f936427fcb1863057 Mon Sep 17 00:00:00 2001 From: Peter Bull Date: Wed, 28 Aug 2024 14:04:16 -0700 Subject: [PATCH 12/14] Revert "weakref finalize instead of __del__" This reverts commit 197a058f59d9791a0cc93e57b24679b4bf86434a. --- cloudpathlib/cache_utils.py | 10 ---------- cloudpathlib/client.py | 27 ++++++++++++--------------- cloudpathlib/cloudpath.py | 36 ++++++++++++++++-------------------- 3 files changed, 28 insertions(+), 45 deletions(-) delete mode 100644 cloudpathlib/cache_utils.py diff --git a/cloudpathlib/cache_utils.py b/cloudpathlib/cache_utils.py deleted file mode 100644 index ebf74ebc..00000000 --- a/cloudpathlib/cache_utils.py +++ /dev/null @@ -1,10 +0,0 @@ -from pathlib import Path -import shutil - - -def _clear_cache(_local: Path) -> None: - if _local.exists(): - if _local.is_file(): - _local.unlink() - else: - shutil.rmtree(_local) diff --git a/cloudpathlib/client.py b/cloudpathlib/client.py index f8ff595a..1b6c32eb 100644 --- a/cloudpathlib/client.py +++ b/cloudpathlib/client.py @@ -5,9 +5,7 @@ import shutil from tempfile import TemporaryDirectory from typing import Generic, Callable, Iterable, Optional, Tuple, TypeVar, Union -import weakref -from .cache_utils import _clear_cache from .cloudpath import CloudImplementation, CloudPath, implementation_registry from .enums import FileCacheMode from .exceptions import InvalidConfigurationException @@ -28,18 +26,6 @@ def decorator(cls: type) -> type: return decorator -def _client_finalizer(file_cache_mode: FileCacheMode, _local_cache_dir) -> None: - if file_cache_mode in [ - FileCacheMode.tmp_dir, - FileCacheMode.close_file, - FileCacheMode.cloudpath_object, - ]: - _clear_cache(_local_cache_dir) - - if _local_cache_dir.exists(): - _local_cache_dir.rmdir() - - class Client(abc.ABC, Generic[BoundedCloudPath]): _cloud_meta: CloudImplementation _default_client = None @@ -96,7 +82,18 @@ def __init__( self.file_cache_mode = file_cache_mode - weakref.finalize(self, _client_finalizer, self.file_cache_mode, self._local_cache_dir) + def __del__(self) -> None: + # remove containing dir, even if a more aggressive strategy + # removed the actual files + if getattr(self, "file_cache_mode", None) in [ + FileCacheMode.tmp_dir, + FileCacheMode.close_file, + FileCacheMode.cloudpath_object, + ]: + self.clear_cache() + + if self._local_cache_dir.exists(): + self._local_cache_dir.rmdir() @classmethod def get_default_client(cls) -> "Client": diff --git a/cloudpathlib/cloudpath.py b/cloudpathlib/cloudpath.py index e2335127..d7bf391b 100644 --- a/cloudpathlib/cloudpath.py +++ b/cloudpathlib/cloudpath.py @@ -11,6 +11,7 @@ _PathParents, ) +import shutil import sys from typing import ( overload, @@ -32,7 +33,6 @@ ) from urllib.parse import urlparse from warnings import warn -import weakref if sys.version_info >= (3, 10): from typing import TypeGuard @@ -58,7 +58,6 @@ def _make_selector(pattern_parts, _flavour, case_sensitive=True): from . import anypath -from .cache_utils import _clear_cache from .exceptions import ( ClientMismatchError, CloudPathFileExistsError, @@ -189,20 +188,6 @@ def __init__(cls, name: str, bases: Tuple[type, ...], dic: Dict[str, Any]) -> No getattr(cls, attr).fget.__doc__ = docstring -def _cloudpath_finalizer(handle: Optional[IO], client: Optional["Client"], _no_prefix: str): - """Use weakref.finalizer instead of __del__ since it is more reliable (does - not wait for garbage collection to actually run). - """ - # make sure that file handle to local path is closed - if handle is not None: - handle.close() - - # ensure file removed from cache when cloudpath object deleted - if client is not None: - if getattr(client, "file_cache_mode", None) == FileCacheMode.cloudpath_object: - _clear_cache(client._local_cache_dir / _no_prefix) - - # Abstract base class class CloudPath(metaclass=CloudPathMeta): """Base class for cloud storage file URIs, in the style of the Python standard library's @@ -257,9 +242,6 @@ def __init__( # track if local has been written to, if so it may need to be uploaded self._dirty = False - # register cache cleanup method when this object is marked for garbage collection - weakref.finalize(self, _cloudpath_finalizer, self._handle, self._client, self._no_prefix) - @property def client(self): if getattr(self, "_client", None) is None: @@ -267,6 +249,16 @@ def client(self): return self._client + def __del__(self) -> None: + # make sure that file handle to local path is closed + if self._handle is not None and self._local.exists(): + self._handle.close() + + # ensure file removed from cache when cloudpath object deleted + client = getattr(self, "_client", None) + if getattr(client, "file_cache_mode", None) == FileCacheMode.cloudpath_object: + self.clear_cache() + def __getstate__(self) -> Dict[str, Any]: state = self.__dict__.copy() @@ -1096,7 +1088,11 @@ def copytree(self, destination, force_overwrite_to_cloud=None, ignore=None): def clear_cache(self): """Removes cache if it exists""" - _clear_cache(self._local) + if self._local.exists(): + if self._local.is_file(): + self._local.unlink() + else: + shutil.rmtree(self._local) # =========== private cloud methods =============== @property From 8045ed044fff2fd80c6a8509f182df541712664e Mon Sep 17 00:00:00 2001 From: Peter Bull Date: Wed, 28 Aug 2024 14:28:52 -0700 Subject: [PATCH 13/14] save time by reducing sleep --- tests/test_cloudpath_upload_copy.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_cloudpath_upload_copy.py b/tests/test_cloudpath_upload_copy.py index 16c12b49..acf5e5ec 100644 --- a/tests/test_cloudpath_upload_copy.py +++ b/tests/test_cloudpath_upload_copy.py @@ -57,7 +57,7 @@ def test_upload_from_file(rig, upload_assets_dir): # to file, file exists to_upload_2 = upload_assets_dir / "upload_2.txt" - sleep(1.5) + sleep(1.1) to_upload_2.touch() # make sure local is newer p.upload_from(to_upload_2) assert p.exists() @@ -70,7 +70,7 @@ def test_upload_from_file(rig, upload_assets_dir): # to file, file exists and is newer; overwrite p.touch() - sleep(1.5) + sleep(1.1) p.upload_from(upload_assets_dir / "upload_1.txt", force_overwrite_to_cloud=True) assert p.exists() assert p.read_text() == "Hello from 1" From 6f51524fff41b18b5fc79b573d0f3995cf432966 Mon Sep 17 00:00:00 2001 From: Peter Bull Date: Wed, 28 Aug 2024 18:31:55 -0700 Subject: [PATCH 14/14] gcs retry --- tests/test_caching.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/test_caching.py b/tests/test_caching.py index a5cb67b7..5bfbbdc2 100644 --- a/tests/test_caching.py +++ b/tests/test_caching.py @@ -3,6 +3,7 @@ from time import sleep from pathlib import Path +from google.api_core.exceptions import TooManyRequests import pytest from tenacity import ( retry, @@ -439,7 +440,7 @@ def test_environment_variables_force_overwrite_to(rig: CloudProviderTestRig, tmp # would raise if not set @retry( - retry=retry_if_exception_type(AssertionError), + retry=retry_if_exception_type((AssertionError, TooManyRequests)), wait=wait_random_exponential(multiplier=0.5, max=5), stop=stop_after_attempt(10), reraise=True, @@ -455,7 +456,9 @@ def _wait_for_cloud_newer(): sleep(0.1) # at least a little different @retry( - retry=retry_if_exception_type((OverwriteNewerLocalError, AssertionError)), + retry=retry_if_exception_type( + (OverwriteNewerLocalError, AssertionError, TooManyRequests) + ), wait=wait_random_exponential(multiplier=0.5, max=5), stop=stop_after_attempt(10), reraise=True,