From 0bc6ad1953371a4695ce50d1e41146944ba9ad16 Mon Sep 17 00:00:00 2001 From: Sidney Mau Date: Mon, 8 Jan 2024 15:52:47 -0800 Subject: [PATCH 1/5] ENH use safe_rm --- eastlake/steps/delete_images.py | 15 ++++++++------- eastlake/steps/delete_meds.py | 5 +++-- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/eastlake/steps/delete_images.py b/eastlake/steps/delete_images.py index 347d39a..d8b0011 100644 --- a/eastlake/steps/delete_images.py +++ b/eastlake/steps/delete_images.py @@ -2,6 +2,7 @@ import os from ..step import Step +from ..utils import safe_rm class DeleteImages(Step): @@ -44,7 +45,7 @@ def execute(self, stash, new_params=None): if filename is not None: if os.path.isfile(filename): self.logger.debug("removing file %s" % filename) - os.remove(filename) + safe_rm(filename) else: self.logger.debug("file %s not found" % filename) else: @@ -57,7 +58,7 @@ def execute(self, stash, new_params=None): if (coadd_file is not None): if os.path.isfile(coadd_file): self.logger.debug("removing file %s" % coadd_file) - os.remove(coadd_file) + safe_rm(coadd_file) # Also check for seg file if self.config["delete_seg"]: @@ -66,17 +67,17 @@ def execute(self, stash, new_params=None): if (seg_file is not None): if os.path.isfile(seg_file): self.logger.debug("removing file %s" % seg_file) - os.remove(seg_file) + safe_rm(seg_file) # Also check for bkg and bkg-rms files bkg_file = coadd_file.replace(".fits", "bkg.fits") if os.path.isfile(bkg_file): self.logger.debug("removing file %s" % bkg_file) - os.remove(bkg_file) + safe_rm(bkg_file) bkg_rms_file = coadd_file.replace(".fits", "bkg-rms.fits") if os.path.isfile(bkg_rms_file): self.logger.debug("removing file %s" % bkg_rms_file) - os.remove(bkg_rms_file) + safe_rm(bkg_rms_file) # Secondly se stuff if self.config["delete_se"]: @@ -89,7 +90,7 @@ def execute(self, stash, new_params=None): for f in img_files: if os.path.isfile(f): self.logger.debug("removing file %s" % f) - os.remove(f) + safe_rm(f) if self.config["delete_se_nwgint"]: self.logger.error("deleting se nwgint images for tile %s" % tilename) @@ -101,6 +102,6 @@ def execute(self, stash, new_params=None): for f in img_files: if os.path.isfile(f): self.logger.debug("removing file %s" % f) - os.remove(f) + safe_rm(f) return 0, stash diff --git a/eastlake/steps/delete_meds.py b/eastlake/steps/delete_meds.py index 4155cf4..621cb4e 100644 --- a/eastlake/steps/delete_meds.py +++ b/eastlake/steps/delete_meds.py @@ -2,6 +2,7 @@ import os from ..step import Step +from ..utils import safe_rm class DeleteMeds(Step): @@ -31,13 +32,13 @@ def execute(self, stash, new_params=None): for m in meds_files: if os.path.isfile(m): self.logger.debug("removing meds file %s" % m) - os.remove(m) + safe_rm(m) meds_files = stash.get_filepaths("pizza_cutter_meds_files", tilename, keyerror=False) if meds_files is not None: for m in meds_files: if os.path.isfile(m): self.logger.debug("removing pizza-cutter meds file %s" % m) - os.remove(m) + safe_rm(m) return 0, stash From cf28945d2ff7e1391ce1891e476c225d56252b8c Mon Sep 17 00:00:00 2001 From: Sidney Mau Date: Mon, 8 Jan 2024 15:52:55 -0800 Subject: [PATCH 2/5] ENH add safe_rmdir --- eastlake/utils.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/eastlake/utils.py b/eastlake/utils.py index 535e265..1be04ee 100644 --- a/eastlake/utils.py +++ b/eastlake/utils.py @@ -64,6 +64,15 @@ def safe_rm(pth, verbose=False): pass +def safe_rmdir(pth, verbose=False): + try: + os.rmdir(pth) + except Exception as e: + if verbose: + print("removing dir %s failed w/ error %r" % (pth, e)) + pass + + def safe_copy(src, dst): safe_mkdir(os.path.dirname(dst)) shutil.copy2(src, dst) From 28dcaaf00ae2a96cfe1168c25949304df1f4ec20 Mon Sep 17 00:00:00 2001 From: Sidney Mau Date: Mon, 8 Jan 2024 15:54:41 -0800 Subject: [PATCH 3/5] ENH use safe_rm --- eastlake/steps/delete_sources.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/eastlake/steps/delete_sources.py b/eastlake/steps/delete_sources.py index b2630e4..5b0bfff 100644 --- a/eastlake/steps/delete_sources.py +++ b/eastlake/steps/delete_sources.py @@ -2,6 +2,7 @@ import os from ..step import Step +from ..utils import safe_rm, safe_rmdir class DeleteSources(Step): @@ -33,7 +34,7 @@ def execute(self, stash, new_params=None): if filename is not None: if os.path.isfile(filename): self.logger.debug("removing file %s" % filename) - os.remove(filename) + safe_rm(filename) else: self.logger.debug("file %s not found" % filename) else: @@ -48,7 +49,7 @@ def execute(self, stash, new_params=None): if (coadd_file is not None): if os.path.isfile(coadd_file): self.logger.debug("removing file %s" % coadd_file) - os.remove(coadd_file) + safe_rm(coadd_file) # Also check for seg file seg_file = stash.get_filepaths( @@ -58,17 +59,17 @@ def execute(self, stash, new_params=None): if (seg_file is not None): if os.path.isfile(seg_file): self.logger.debug("removing file %s" % seg_file) - os.remove(seg_file) + safe_rm(seg_file) # Also check for bkg and bkg-rms files bkg_file = coadd_file.replace(".fits", "bkg.fits") if os.path.isfile(bkg_file): self.logger.debug("removing file %s" % bkg_file) - os.remove(bkg_file) + safe_rm(bkg_file) bkg_rms_file = coadd_file.replace(".fits", "bkg-rms.fits") if os.path.isfile(bkg_rms_file): self.logger.debug("removing file %s" % bkg_rms_file) - os.remove(bkg_rms_file) + safe_rm(bkg_rms_file) self.logger.error("deleting se images for tile %s" % tilename) for band in stash["bands"]: @@ -79,7 +80,7 @@ def execute(self, stash, new_params=None): for f in img_files: if os.path.isfile(f): self.logger.debug("removing file %s" % f) - os.remove(f) + safe_rm(f) self.logger.error("deleting se nwgint images for tile %s" % tilename) for band in stash["bands"]: @@ -90,7 +91,7 @@ def execute(self, stash, new_params=None): for f in img_files: if os.path.isfile(f): self.logger.debug("removing file %s" % f) - os.remove(f) + safe_rm(f) self.logger.error("deleting as much as we can for tile %s" % tilename) for band in stash["bands"]: @@ -105,7 +106,7 @@ def execute(self, stash, new_params=None): for t in totry: if os.path.isfile(t): self.logger.debug("removing file %s" % t) - os.remove(t) + safe_rm(t) if k == "src_info": for srci in pyml["src_info"]: From ce34c7d4c83a7e076eaf748625fe0b4661598a16 Mon Sep 17 00:00:00 2001 From: Sidney Mau Date: Mon, 8 Jan 2024 15:57:47 -0800 Subject: [PATCH 4/5] ENH remove psf links --- eastlake/steps/delete_sources.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/eastlake/steps/delete_sources.py b/eastlake/steps/delete_sources.py index 5b0bfff..acda7b5 100644 --- a/eastlake/steps/delete_sources.py +++ b/eastlake/steps/delete_sources.py @@ -22,6 +22,7 @@ def __init__(self, config, base_dir, name="delete_sources", def execute(self, stash, new_params=None): + base_dir = stash["base_dir"] tilenames = stash["tilenames"] for tilename in tilenames: if tilename in self.config["save_tilenames"]: @@ -120,6 +121,26 @@ def execute(self, stash, new_params=None): for t in totry: if os.path.isfile(t): self.logger.debug("removing file %s" % t) - os.remove(t) + safe_rm(t) + + self.logger.error("removing psf links for %s" % tilename) + + psf_link = os.path.join( + base_dir, stash["desrun"], tilename, "psfs", + os.path.basename(pyml["psf_path"]) + ) + safe_rm(psf_link) + + for sri in pyml["src_info"]: + psf_link = os.path.join( + base_dir, stash["desrun"], tilename, "psfs", + os.path.basename(sri["psf_path"]) + ) + safe_rm(psf_link) + + psf_path = os.path.join( + base_dir, stash["desrun"], tilename, "psfs", + ) + safe_rmdir(psf_path) return 0, stash From 032e7b01b0c29f26846e061d715faa0805fd2842 Mon Sep 17 00:00:00 2001 From: Sidney Mau Date: Mon, 8 Jan 2024 15:58:07 -0800 Subject: [PATCH 5/5] ENH remove empty dirs --- eastlake/steps/delete_sources.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/eastlake/steps/delete_sources.py b/eastlake/steps/delete_sources.py index acda7b5..e78b623 100644 --- a/eastlake/steps/delete_sources.py +++ b/eastlake/steps/delete_sources.py @@ -143,4 +143,12 @@ def execute(self, stash, new_params=None): ) safe_rmdir(psf_path) + self.logger.error("deleting empty dirs") + + for root, dirs, files in os.walk(base_dir, topdown=False): + for name in dirs: + full_dir = os.path.join(root, name) + if len(os.listdir(full_dir)) == 0: + safe_rmdir(full_dir) + return 0, stash