Skip to content

Commit

Permalink
Merge pull request #65 from des-science/64
Browse files Browse the repository at this point in the history
  • Loading branch information
@beckermr
beckermr authored Jan 9, 2024
2 parents 2462de9 + 032e7b0 commit 8ad846c
Show file tree
Hide file tree
Showing 4 changed files with 59 additions and 18 deletions.
15 changes: 8 additions & 7 deletions eastlake/steps/delete_images.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
import os

from ..step import Step
from ..utils import safe_rm


class DeleteImages(Step):
Expand Down Expand Up @@ -44,7 +45,7 @@ def execute(self, stash, new_params=None):
if filename is not None:
if os.path.isfile(filename):
self.logger.debug("removing file %s" % filename)
os.remove(filename)
safe_rm(filename)
else:
self.logger.debug("file %s not found" % filename)
else:
Expand All @@ -57,7 +58,7 @@ def execute(self, stash, new_params=None):
if (coadd_file is not None):
if os.path.isfile(coadd_file):
self.logger.debug("removing file %s" % coadd_file)
os.remove(coadd_file)
safe_rm(coadd_file)

# Also check for seg file
if self.config["delete_seg"]:
Expand All @@ -66,17 +67,17 @@ def execute(self, stash, new_params=None):
if (seg_file is not None):
if os.path.isfile(seg_file):
self.logger.debug("removing file %s" % seg_file)
os.remove(seg_file)
safe_rm(seg_file)

# Also check for bkg and bkg-rms files
bkg_file = coadd_file.replace(".fits", "bkg.fits")
if os.path.isfile(bkg_file):
self.logger.debug("removing file %s" % bkg_file)
os.remove(bkg_file)
safe_rm(bkg_file)
bkg_rms_file = coadd_file.replace(".fits", "bkg-rms.fits")
if os.path.isfile(bkg_rms_file):
self.logger.debug("removing file %s" % bkg_rms_file)
os.remove(bkg_rms_file)
safe_rm(bkg_rms_file)

# Secondly se stuff
if self.config["delete_se"]:
Expand All @@ -89,7 +90,7 @@ def execute(self, stash, new_params=None):
for f in img_files:
if os.path.isfile(f):
self.logger.debug("removing file %s" % f)
os.remove(f)
safe_rm(f)

if self.config["delete_se_nwgint"]:
self.logger.error("deleting se nwgint images for tile %s" % tilename)
Expand All @@ -101,6 +102,6 @@ def execute(self, stash, new_params=None):
for f in img_files:
if os.path.isfile(f):
self.logger.debug("removing file %s" % f)
os.remove(f)
safe_rm(f)

return 0, stash
5 changes: 3 additions & 2 deletions eastlake/steps/delete_meds.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
import os

from ..step import Step
from ..utils import safe_rm


class DeleteMeds(Step):
Expand Down Expand Up @@ -31,13 +32,13 @@ def execute(self, stash, new_params=None):
for m in meds_files:
if os.path.isfile(m):
self.logger.debug("removing meds file %s" % m)
os.remove(m)
safe_rm(m)

meds_files = stash.get_filepaths("pizza_cutter_meds_files", tilename, keyerror=False)
if meds_files is not None:
for m in meds_files:
if os.path.isfile(m):
self.logger.debug("removing pizza-cutter meds file %s" % m)
os.remove(m)
safe_rm(m)

return 0, stash
48 changes: 39 additions & 9 deletions eastlake/steps/delete_sources.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
import os

from ..step import Step
from ..utils import safe_rm, safe_rmdir


class DeleteSources(Step):
Expand All @@ -21,6 +22,7 @@ def __init__(self, config, base_dir, name="delete_sources",

def execute(self, stash, new_params=None):

base_dir = stash["base_dir"]
tilenames = stash["tilenames"]
for tilename in tilenames:
if tilename in self.config["save_tilenames"]:
Expand All @@ -33,7 +35,7 @@ def execute(self, stash, new_params=None):
if filename is not None:
if os.path.isfile(filename):
self.logger.debug("removing file %s" % filename)
os.remove(filename)
safe_rm(filename)
else:
self.logger.debug("file %s not found" % filename)
else:
Expand All @@ -48,7 +50,7 @@ def execute(self, stash, new_params=None):
if (coadd_file is not None):
if os.path.isfile(coadd_file):
self.logger.debug("removing file %s" % coadd_file)
os.remove(coadd_file)
safe_rm(coadd_file)

# Also check for seg file
seg_file = stash.get_filepaths(
Expand All @@ -58,17 +60,17 @@ def execute(self, stash, new_params=None):
if (seg_file is not None):
if os.path.isfile(seg_file):
self.logger.debug("removing file %s" % seg_file)
os.remove(seg_file)
safe_rm(seg_file)

# Also check for bkg and bkg-rms files
bkg_file = coadd_file.replace(".fits", "bkg.fits")
if os.path.isfile(bkg_file):
self.logger.debug("removing file %s" % bkg_file)
os.remove(bkg_file)
safe_rm(bkg_file)
bkg_rms_file = coadd_file.replace(".fits", "bkg-rms.fits")
if os.path.isfile(bkg_rms_file):
self.logger.debug("removing file %s" % bkg_rms_file)
os.remove(bkg_rms_file)
safe_rm(bkg_rms_file)

self.logger.error("deleting se images for tile %s" % tilename)
for band in stash["bands"]:
Expand All @@ -79,7 +81,7 @@ def execute(self, stash, new_params=None):
for f in img_files:
if os.path.isfile(f):
self.logger.debug("removing file %s" % f)
os.remove(f)
safe_rm(f)

self.logger.error("deleting se nwgint images for tile %s" % tilename)
for band in stash["bands"]:
Expand All @@ -90,7 +92,7 @@ def execute(self, stash, new_params=None):
for f in img_files:
if os.path.isfile(f):
self.logger.debug("removing file %s" % f)
os.remove(f)
safe_rm(f)

self.logger.error("deleting as much as we can for tile %s" % tilename)
for band in stash["bands"]:
Expand All @@ -105,7 +107,7 @@ def execute(self, stash, new_params=None):
for t in totry:
if os.path.isfile(t):
self.logger.debug("removing file %s" % t)
os.remove(t)
safe_rm(t)

if k == "src_info":
for srci in pyml["src_info"]:
Expand All @@ -119,6 +121,34 @@ def execute(self, stash, new_params=None):
for t in totry:
if os.path.isfile(t):
self.logger.debug("removing file %s" % t)
os.remove(t)
safe_rm(t)

self.logger.error("removing psf links for %s" % tilename)

psf_link = os.path.join(
base_dir, stash["desrun"], tilename, "psfs",
os.path.basename(pyml["psf_path"])
)
safe_rm(psf_link)

for sri in pyml["src_info"]:
psf_link = os.path.join(
base_dir, stash["desrun"], tilename, "psfs",
os.path.basename(sri["psf_path"])
)
safe_rm(psf_link)

psf_path = os.path.join(
base_dir, stash["desrun"], tilename, "psfs",
)
safe_rmdir(psf_path)

self.logger.error("deleting empty dirs")

for root, dirs, files in os.walk(base_dir, topdown=False):
for name in dirs:
full_dir = os.path.join(root, name)
if len(os.listdir(full_dir)) == 0:
safe_rmdir(full_dir)

return 0, stash
9 changes: 9 additions & 0 deletions eastlake/utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,15 @@ def safe_rm(pth, verbose=False):
pass


def safe_rmdir(pth, verbose=False):
try:
os.rmdir(pth)
except Exception as e:
if verbose:
print("removing dir %s failed w/ error %r" % (pth, e))
pass


def safe_copy(src, dst):
safe_mkdir(os.path.dirname(dst))
shutil.copy2(src, dst)
Expand Down

0 comments on commit 8ad846c

Please sign in to comment.