From 66cdccd7e2861aad59a585f6f727b090ae46eb80 Mon Sep 17 00:00:00 2001 From: James Davies Date: Wed, 6 Mar 2024 09:54:07 +0100 Subject: [PATCH 1/3] Use datamodels to write out optional mask and median --- src/snowblind/rc_selfcal.py | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/src/snowblind/rc_selfcal.py b/src/snowblind/rc_selfcal.py index 2b8f171..a4b1d92 100644 --- a/src/snowblind/rc_selfcal.py +++ b/src/snowblind/rc_selfcal.py @@ -1,3 +1,4 @@ +from os.path import commonprefix import warnings from astropy.stats import sigma_clipped_stats @@ -27,8 +28,8 @@ class RcSelfCalStep(Step): insert it anywhere in the level3 pipeline before resample. """ spec = """ - threshold = float(default=3.0) # threshold in sigma to flag hot pixels above median - save_mask = boolean(default=False) # write out per-detector bad-pixel masks + threshold = float(default=3.0) # threshold in sigma to flag hot pixels above local background + save_mask = boolean(default=False) # write out per-detector bad-pixel mask and median output_use_model = boolean(default=True) output_use_index = boolean(default=False) """ @@ -55,16 +56,13 @@ def process(self, input_data): mask, median = self.create_hotpixel_mask(image_stack) self.log.info(f"Flagged {mask.sum()} pixels with {self.threshold} sigma") if self.save_mask: - fits.HDUList( - fits.PrimaryHDU( - data=mask.astype(np.uint8) - ) - ).writeto(f"{detector.lower()}_rcflag_mask.fits", overwrite=True) - fits.HDUList( - fits.PrimaryHDU( - data=median - ) - ).writeto(f"{detector.lower()}_rcflag_median.fits", overwrite=True) + filename_prefix = f"{commonprefix([f.meta.filename for f in models])}_{detector.lower()}_{self.class_alias}" + mask_model = datamodels.MaskModel(data=mask.astype(np.uint8)) + mask_model.meta.filename = f"{filename_prefix}.fits" + self.save_model(mask_model, suffix="mask", force=True) + median_model = datamodels.ImageModel(data=median) + median_model.meta.filename = f"{filename_prefix}.fits" + self.save_model(median_model, suffix="median", force=True) for result in results: if result.meta.instrument.detector == detector: From 10133c43bc9c4bd3aed6ae64301e690ba1eb96e8 Mon Sep 17 00:00:00 2001 From: James Davies Date: Wed, 6 Mar 2024 10:21:53 +0100 Subject: [PATCH 2/3] Rename and move rc_selfcal modules --- src/snowblind/__init__.py | 6 +++--- src/snowblind/{rc_selfcal.py => selfcal.py} | 14 +++++++------- tests/{test_rc_selfcal.py => test_selfcal.py} | 14 +++++++------- 3 files changed, 17 insertions(+), 17 deletions(-) rename src/snowblind/{rc_selfcal.py => selfcal.py} (92%) rename tests/{test_rc_selfcal.py => test_selfcal.py} (76%) diff --git a/src/snowblind/__init__.py b/src/snowblind/__init__.py index 3b545b5..402c34c 100644 --- a/src/snowblind/__init__.py +++ b/src/snowblind/__init__.py @@ -2,7 +2,7 @@ from .snowblind import SnowblindStep from .jump_plus import JumpPlusStep -from .rc_selfcal import RcSelfCalStep +from .selfcal import OpenPixelStep from .persist import PersistenceFlagStep @@ -16,7 +16,7 @@ '__version__', 'SnowblindStep', 'JumpPlusStep', - 'RcSelfCalStep', + 'OpenPixelStep', 'PersistenceFlagStep', ] @@ -27,6 +27,6 @@ def _get_steps(): return [ ("snowblind.SnowblindStep", 'snowblind', False), ("snowblind.JumpPlusStep", 'jump_plus', False), - ("snowblind.RcSelfCalStep", 'rc_selfcal', False), + ("snowblind.OpenPixelStep", 'open_pixel', False), ("snowblind.PersistenceFlagStep", 'persist', False), ] diff --git a/src/snowblind/rc_selfcal.py b/src/snowblind/selfcal.py similarity index 92% rename from src/snowblind/rc_selfcal.py rename to src/snowblind/selfcal.py index a4b1d92..75dc6f8 100644 --- a/src/snowblind/rc_selfcal.py +++ b/src/snowblind/selfcal.py @@ -2,18 +2,18 @@ import warnings from astropy.stats import sigma_clipped_stats -from astropy.io import fits import numpy as np from jwst import datamodels from jwst.stpipe import Step -RC = datamodels.dqflags.pixel["RC"] +OPEN = datamodels.dqflags.pixel["OPEN"] +ADJ_OPEN = datamodels.dqflags.pixel["ADJ_OPEN"] DO_NOT_USE = datamodels.dqflags.pixel["DO_NOT_USE"] -class RcSelfCalStep(Step): - """Removes cross-shaped and other defects caused by RC-type bad pixels in NIR detectors +class OpenPixelStep(Step): + """Flags cross-shaped and hot pixel defects caused by open pixels in NIR detectors Input is an assocation (or glob pattern of files) of all images in visit or program ID on which ones wishes to do a self-cal. These are split into separate detector stacks, @@ -22,7 +22,7 @@ class RcSelfCalStep(Step): stage 3 pipelines such as tweakreg, skymatch and outlier detection. Like outlier_detection, the input and output science images are the same, and only the - data quality (DQ) array has new pixels flagged as DO_NOT_USE and RC. + data quality (DQ) array has new pixels flagged as DO_NOT_USE and ADJ_OPEN. This should be run after flatfielding is finished in image2 pipeline. It is fine to insert it anywhere in the level3 pipeline before resample. @@ -34,7 +34,7 @@ class RcSelfCalStep(Step): output_use_index = boolean(default=False) """ - class_alias = "rc_selfcal" + class_alias = "open_pixel" def process(self, input_data): with datamodels.open(input_data) as images: @@ -66,7 +66,7 @@ def process(self, input_data): for result in results: if result.meta.instrument.detector == detector: - result.dq |= (mask * (DO_NOT_USE | RC)).astype(np.uint32) + result.dq |= (mask * (DO_NOT_USE | ADJ_OPEN)).astype(np.uint32) return results diff --git a/tests/test_rc_selfcal.py b/tests/test_selfcal.py similarity index 76% rename from tests/test_rc_selfcal.py rename to tests/test_selfcal.py index 0dbd950..d85519c 100644 --- a/tests/test_rc_selfcal.py +++ b/tests/test_selfcal.py @@ -1,16 +1,16 @@ import numpy as np from jwst import datamodels -from snowblind import RcSelfCalStep +from snowblind import OpenPixelStep -RC = datamodels.dqflags.pixel["RC"] +ADJ_OPEN = datamodels.dqflags.pixel["ADJ_OPEN"] DO_NOT_USE = datamodels.dqflags.pixel["DO_NOT_USE"] GOOD = datamodels.dqflags.pixel["GOOD"] def test_init(): - step = RcSelfCalStep(threshold=4.5) + step = OpenPixelStep(threshold=4.5) assert step.threshold == 4.5 @@ -41,11 +41,11 @@ def test_call(): image.data[8, 8] += 3 * stddev # Run the step and see if they're recovered - results = RcSelfCalStep.call(images, threshold=3.0) + results = OpenPixelStep.call(images, threshold=3.0) for result in results: - assert result.dq[2, 2] == RC | DO_NOT_USE - assert result.dq[3, 5] == RC | DO_NOT_USE - assert result.dq[8, 8] == RC | DO_NOT_USE + assert result.dq[2, 2] == ADJ_OPEN | DO_NOT_USE + assert result.dq[3, 5] == ADJ_OPEN | DO_NOT_USE + assert result.dq[8, 8] == ADJ_OPEN | DO_NOT_USE assert result.dq[5, 5] == GOOD From 6cf7b300f816f195dcce98b9ef3fd9f0f758cdb4 Mon Sep 17 00:00:00 2001 From: James Davies Date: Wed, 6 Mar 2024 10:31:18 +0100 Subject: [PATCH 3/3] Update README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 103c68f..b3c241e 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ Algorithms for cleaning JWST data. - `SnowblindStep`: mask cosmic ray showers and snowballs - `JumpPlusStep`: Propagate JUMP_DET and SATURATED flags in GROUPDQ properly for frame-averaged groups - `PersistenceFlagStep`: flag pixels effected by persistence exposure-to-exposure - - `RcSelfCalStep`: flag new hot pixels, open pixels or RC pixels + - `OpenPixelStep`: flag new open pixels, hot pixels, or open adjacent pixels ## Installation @@ -31,6 +31,7 @@ In Python, we can insert `SnowblindStep` and `JumpPlusStep` after `JumpStep` as steps = { "jump": { "save_results": True, + "flag_large_events": False, "post_hooks": [ "snowblind.SnowblindStep", "snowblind.JumpPlusStep",