diff --git a/eastlake/steps/delete_images.py b/eastlake/steps/delete_images.py index 347d39a..d8b0011 100644 --- a/eastlake/steps/delete_images.py +++ b/eastlake/steps/delete_images.py @@ -2,6 +2,7 @@ import os from ..step import Step +from ..utils import safe_rm class DeleteImages(Step): @@ -44,7 +45,7 @@ def execute(self, stash, new_params=None): if filename is not None: if os.path.isfile(filename): self.logger.debug("removing file %s" % filename) - os.remove(filename) + safe_rm(filename) else: self.logger.debug("file %s not found" % filename) else: @@ -57,7 +58,7 @@ def execute(self, stash, new_params=None): if (coadd_file is not None): if os.path.isfile(coadd_file): self.logger.debug("removing file %s" % coadd_file) - os.remove(coadd_file) + safe_rm(coadd_file) # Also check for seg file if self.config["delete_seg"]: @@ -66,17 +67,17 @@ def execute(self, stash, new_params=None): if (seg_file is not None): if os.path.isfile(seg_file): self.logger.debug("removing file %s" % seg_file) - os.remove(seg_file) + safe_rm(seg_file) # Also check for bkg and bkg-rms files bkg_file = coadd_file.replace(".fits", "bkg.fits") if os.path.isfile(bkg_file): self.logger.debug("removing file %s" % bkg_file) - os.remove(bkg_file) + safe_rm(bkg_file) bkg_rms_file = coadd_file.replace(".fits", "bkg-rms.fits") if os.path.isfile(bkg_rms_file): self.logger.debug("removing file %s" % bkg_rms_file) - os.remove(bkg_rms_file) + safe_rm(bkg_rms_file) # Secondly se stuff if self.config["delete_se"]: @@ -89,7 +90,7 @@ def execute(self, stash, new_params=None): for f in img_files: if os.path.isfile(f): self.logger.debug("removing file %s" % f) - os.remove(f) + safe_rm(f) if self.config["delete_se_nwgint"]: self.logger.error("deleting se nwgint images for tile %s" % tilename) @@ -101,6 +102,6 @@ def execute(self, stash, new_params=None): for f in img_files: if os.path.isfile(f): self.logger.debug("removing file %s" % f) - os.remove(f) + safe_rm(f) return 0, stash diff --git a/eastlake/steps/delete_meds.py b/eastlake/steps/delete_meds.py index 4155cf4..621cb4e 100644 --- a/eastlake/steps/delete_meds.py +++ b/eastlake/steps/delete_meds.py @@ -2,6 +2,7 @@ import os from ..step import Step +from ..utils import safe_rm class DeleteMeds(Step): @@ -31,13 +32,13 @@ def execute(self, stash, new_params=None): for m in meds_files: if os.path.isfile(m): self.logger.debug("removing meds file %s" % m) - os.remove(m) + safe_rm(m) meds_files = stash.get_filepaths("pizza_cutter_meds_files", tilename, keyerror=False) if meds_files is not None: for m in meds_files: if os.path.isfile(m): self.logger.debug("removing pizza-cutter meds file %s" % m) - os.remove(m) + safe_rm(m) return 0, stash diff --git a/eastlake/steps/delete_sources.py b/eastlake/steps/delete_sources.py index b2630e4..e78b623 100644 --- a/eastlake/steps/delete_sources.py +++ b/eastlake/steps/delete_sources.py @@ -2,6 +2,7 @@ import os from ..step import Step +from ..utils import safe_rm, safe_rmdir class DeleteSources(Step): @@ -21,6 +22,7 @@ def __init__(self, config, base_dir, name="delete_sources", def execute(self, stash, new_params=None): + base_dir = stash["base_dir"] tilenames = stash["tilenames"] for tilename in tilenames: if tilename in self.config["save_tilenames"]: @@ -33,7 +35,7 @@ def execute(self, stash, new_params=None): if filename is not None: if os.path.isfile(filename): self.logger.debug("removing file %s" % filename) - os.remove(filename) + safe_rm(filename) else: self.logger.debug("file %s not found" % filename) else: @@ -48,7 +50,7 @@ def execute(self, stash, new_params=None): if (coadd_file is not None): if os.path.isfile(coadd_file): self.logger.debug("removing file %s" % coadd_file) - os.remove(coadd_file) + safe_rm(coadd_file) # Also check for seg file seg_file = stash.get_filepaths( @@ -58,17 +60,17 @@ def execute(self, stash, new_params=None): if (seg_file is not None): if os.path.isfile(seg_file): self.logger.debug("removing file %s" % seg_file) - os.remove(seg_file) + safe_rm(seg_file) # Also check for bkg and bkg-rms files bkg_file = coadd_file.replace(".fits", "bkg.fits") if os.path.isfile(bkg_file): self.logger.debug("removing file %s" % bkg_file) - os.remove(bkg_file) + safe_rm(bkg_file) bkg_rms_file = coadd_file.replace(".fits", "bkg-rms.fits") if os.path.isfile(bkg_rms_file): self.logger.debug("removing file %s" % bkg_rms_file) - os.remove(bkg_rms_file) + safe_rm(bkg_rms_file) self.logger.error("deleting se images for tile %s" % tilename) for band in stash["bands"]: @@ -79,7 +81,7 @@ def execute(self, stash, new_params=None): for f in img_files: if os.path.isfile(f): self.logger.debug("removing file %s" % f) - os.remove(f) + safe_rm(f) self.logger.error("deleting se nwgint images for tile %s" % tilename) for band in stash["bands"]: @@ -90,7 +92,7 @@ def execute(self, stash, new_params=None): for f in img_files: if os.path.isfile(f): self.logger.debug("removing file %s" % f) - os.remove(f) + safe_rm(f) self.logger.error("deleting as much as we can for tile %s" % tilename) for band in stash["bands"]: @@ -105,7 +107,7 @@ def execute(self, stash, new_params=None): for t in totry: if os.path.isfile(t): self.logger.debug("removing file %s" % t) - os.remove(t) + safe_rm(t) if k == "src_info": for srci in pyml["src_info"]: @@ -119,6 +121,34 @@ def execute(self, stash, new_params=None): for t in totry: if os.path.isfile(t): self.logger.debug("removing file %s" % t) - os.remove(t) + safe_rm(t) + + self.logger.error("removing psf links for %s" % tilename) + + psf_link = os.path.join( + base_dir, stash["desrun"], tilename, "psfs", + os.path.basename(pyml["psf_path"]) + ) + safe_rm(psf_link) + + for sri in pyml["src_info"]: + psf_link = os.path.join( + base_dir, stash["desrun"], tilename, "psfs", + os.path.basename(sri["psf_path"]) + ) + safe_rm(psf_link) + + psf_path = os.path.join( + base_dir, stash["desrun"], tilename, "psfs", + ) + safe_rmdir(psf_path) + + self.logger.error("deleting empty dirs") + + for root, dirs, files in os.walk(base_dir, topdown=False): + for name in dirs: + full_dir = os.path.join(root, name) + if len(os.listdir(full_dir)) == 0: + safe_rmdir(full_dir) return 0, stash diff --git a/eastlake/utils.py b/eastlake/utils.py index 535e265..1be04ee 100644 --- a/eastlake/utils.py +++ b/eastlake/utils.py @@ -64,6 +64,15 @@ def safe_rm(pth, verbose=False): pass +def safe_rmdir(pth, verbose=False): + try: + os.rmdir(pth) + except Exception as e: + if verbose: + print("removing dir %s failed w/ error %r" % (pth, e)) + pass + + def safe_copy(src, dst): safe_mkdir(os.path.dirname(dst)) shutil.copy2(src, dst)