JP-3682: split outlier detection into separate steps for each mode

docs/jwst/outlier_detection/outlier_detection.rst

@@ -0,0 +1,75 @@
+Outlier Detection Transition Guide
+==================================
+
+As of jwst release 1.18, the outlier detection step has been replaced by
+separate steps for the various observing modes. This page serves as a transition
+guide for migrating to use the new steps.
+
+The new steps and the exposure types processed through those steps are:
+
+.. list-table:: Outlier Detection Steps
+   :header-rows: 1
+
+   * - Step
+     - Exposure Types
+   * - `OutlierDetectionCoronStep`
+     - 'MIR_LYOT', 'MIR_4QPM', 'NRC_CORON'
+   * - `OutlierDetectionIFUStep`
+     - 'MIR_MRS', 'NRS_IFU'
+   * - `OutlierDetectionImagingStep`
+     - 'FGS_IMAGE', 'MIR_IMAGE', 'NRC_IMAGE', 'NIS_IMAGE'
+   * - `OutlierDetectionSpecStep`
+     - 'MIR_LRS-FIXEDSLIT', 'NRS_FIXEDSLIT', 'NRS_MSASPEC'
+   * - `OutlierDetectionTSOStep`
+     - 'MIR_LRS-SLITLESS', 'NRC_TSGRISM', 'NIS_SOSS', 'NRS_BRIGHTOBJ', 'NRC_TSIMAGE'
+
+Command-line syntax
+-------------------
+
+To run the individual step on its own, use the following command line syntax:
+
+::
+
+   strun outlier_detection_coron input.fits
+
+instead of the previous syntax:
+
+::
+
+   strun outlier_detection input.fits # NO LONGER WORKS
+
+
+The new steps have the same optional arguments as the original step, but only
+the ones relevant to that observing mode.
+For example, the `OutlierDetectionCoronStep` has the
+`--save_intermediate_results`, `--good_bits`, `--snr`, and `--maskpt` arguments,
+but no longer has the `--kernel` argument, which didn't have any effect
+for coronagraphic observations in the first place.
+
+Running the new steps within a pipeline works the same as before. To set options
+for the new steps within a pipeline, use the `--steps.<step_name>` syntax, e.g.
+
+::
+
+   strun calwebb_image3 --steps.outlier_detection_coron.save_intermediate_results=True input.fits
+
+Note that for `calwebb_spec3`, the outlier detection step is different for IFU data than
+for slit-like data. For IFU data, use `--steps.outlier_detection_ifu`, and for fixed slit
+and MSA data, use `--steps.outlier_detection_spec`.
+
+Python syntax
+-------------
+The new steps can be run in Python using the following syntax:
+
+::
+
+   from jwst.outlier_detection_coron import OutlierDetectionCoronStep
+   step = OutlierDetectionCoronStep()
+   step.call(input.fits, save_intermediate_results=True)
+
+The new pipelines can be run in Python using, e.g.,:
+
+::
+
+   from jwst.pipeline import calwebb_image3
+   calwebb_image3(input.fits, steps={'outlier_detection_coron': {'save_intermediate_results': True}})

docs/jwst/package_index.rst

@@ -45,6 +45,7 @@ Package Index
    mrs_imatch/index.rst
    msaflagopen/index.rst
    nsclean/index.rst
+   outlier_detection/outlier_detection.rst
    outlier_detection_coron/index.rst
    outlier_detection_ifu/index.rst
    outlier_detection_imaging/index.rst

jwst/outlier_detection/tests/test_outlier_detection.py

@@ -374,7 +374,7 @@ def test_outlier_step_spec(tmp_cwd, tmp_path, resample, save_intermediate):
         expected_intermediate = 0
     for dirname in [output_dir, tmp_cwd]:
         all_files = glob(os.path.join(dirname, '*.fits'))
-        result_files = glob(os.path.join(dirname, '*outlierdetectionspecstep.fits'))
+        result_files = glob(os.path.join(dirname, '*crf.fits'))
         i2d_files = glob(os.path.join(dirname, '*i2d*.fits'))
         s2d_files = glob(os.path.join(dirname, '*outlier_s2d.fits'))
         median_files = glob(os.path.join(dirname, '*median.fits'))
@@ -484,7 +484,7 @@ def test_outlier_step_on_disk(three_sci_as_asn, tmp_cwd):
     dirname = tmp_cwd
     all_files = glob(os.path.join(dirname, '*.fits'))
     input_files = glob(os.path.join(dirname, '*_cal.fits'))
-    result_files = glob(os.path.join(dirname, '*outlierdetectionimagingstep.fits'))
+    result_files = glob(os.path.join(dirname, '*crf.fits'))
     i2d_files = glob(os.path.join(dirname, '*i2d*.fits'))
     s2d_files = glob(os.path.join(dirname, '*outlier_s2d.fits'))
     median_files = glob(os.path.join(dirname, '*median.fits'))

jwst/pipeline/calwebb_coron3.py

@@ -58,7 +58,7 @@ class Coron3Pipeline(Pipeline):
     #. stack_refs (assemble reference PSF inputs)
     #. align_refs (align reference PSFs to target images)
     #. klip (PSF subtraction using the KLIP algorithm)
-    #. outlier_detection (flag outliers)
+    #. outlier_detection_coron (flag outliers)
     #. resample (image combination and resampling)
 
     """
@@ -74,7 +74,7 @@ class Coron3Pipeline(Pipeline):
         'stack_refs': stack_refs_step.StackRefsStep,
         'align_refs': align_refs_step.AlignRefsStep,
         'klip': klip_step.KlipStep,
-        'outlier_detection': outlier_detection_coron_step.OutlierDetectionCoronStep,
+        'outlier_detection_coron': outlier_detection_coron_step.OutlierDetectionCoronStep,
         'resample': resample_step.ResampleStep
     }
 
@@ -110,13 +110,13 @@ def process(self, user_input):
             members_by_type[member['exptype'].lower()].append(member['expname'])
 
         # Set up required output products and formats
-        self.outlier_detection.save_results = self.save_results
+        self.outlier_detection_coron.save_results = self.save_results
         self.resample.blendheaders = False
 
         # Save the original outlier_detection.skip setting from the
         # input, because it may get toggled off within loops for
         # processing individual inputs
-        skip_outlier_detection = self.outlier_detection.skip
+        skip_outlier_detection = self.outlier_detection_coron.skip
 
         # Extract lists of all the PSF and science target members
         psf_files = members_by_type['psf']
@@ -149,10 +149,10 @@ def process(self, user_input):
         # Perform outlier detection on the PSFs.
         if not skip_outlier_detection:
             for model in psf_models:
-                self.outlier_detection.run(model)
+                self.outlier_detection_coron.run(model)
                 # step may have been skipped for this model;
                 # turn back on for next model
-                self.outlier_detection.skip = False
+                self.outlier_detection_coron.skip = False
         else:
             self.log.info('Outlier detection skipped for PSF\'s')
 
@@ -173,10 +173,10 @@ def process(self, user_input):
 
                 # Remove outliers from the target
                 if not skip_outlier_detection:
-                    target = self.outlier_detection.run(target)
+                    target = self.outlier_detection_coron.run(target)
                     # step may have been skipped for this model;
                     # turn back on for next model
-                    self.outlier_detection.skip = False
+                    self.outlier_detection_coron.skip = False
 
                 # Call align_refs
                 psf_aligned = self.align_refs.run(target, psf_stack)

jwst/pipeline/calwebb_image3.py

@@ -25,7 +25,7 @@ class Image3Pipeline(Pipeline):
         assign_mtwcs
         tweakreg
         skymatch
-        outlier_detection
+        outlier_detection_imaging
         resample
         source_catalog
     """
@@ -41,7 +41,7 @@ class Image3Pipeline(Pipeline):
         'assign_mtwcs': assign_mtwcs_step.AssignMTWcsStep,
         'tweakreg': tweakreg_step.TweakRegStep,
         'skymatch': skymatch_step.SkyMatchStep,
-        'outlier_detection': outlier_detection_imaging_step.OutlierDetectionImagingStep,
+        'outlier_detection_imaging': outlier_detection_imaging_step.OutlierDetectionImagingStep,
         'resample': resample_step.ResampleStep,
         'source_catalog': source_catalog_step.SourceCatalogStep
     }
@@ -58,7 +58,7 @@ def process(self, input_data):
         self.log.info('Starting calwebb_image3 ...')
 
         # Configure settings for saving results files
-        self.outlier_detection.save_results = self.save_results
+        self.outlier_detection_imaging.save_results = self.save_results
 
         self.resample.suffix = 'i2d'
         self.resample.save_results = self.save_results
@@ -87,7 +87,7 @@ def process(self, input_data):
                 input_models = self.tweakreg.run(input_models)
 
             input_models = self.skymatch.run(input_models)
-            input_models = self.outlier_detection.run(input_models)
+            input_models = self.outlier_detection_imaging.run(input_models)
 
         elif self.skymatch.skymethod == 'match':
             self.log.warning("Turning 'skymatch' step off for a single "

jwst/pipeline/calwebb_tso3.py

@@ -40,7 +40,7 @@ class Tso3Pipeline(Pipeline):
     spec = ""
 
     # Define alias to steps
-    step_defs = {'outlier_detection': outlier_detection_tso_step.OutlierDetectionTSOStep,
+    step_defs = {'outlier_detection_tso': outlier_detection_tso_step.OutlierDetectionTSOStep,
                  'tso_photometry': tso_photometry_step.TSOPhotometryStep,
                  'pixel_replace': pixel_replace_step.PixelReplaceStep,
                  'extract_1d': extract_1d_step.Extract1dStep,
@@ -85,16 +85,16 @@ def process(self, input):
 
             # Can't do outlier detection if there isn't a stack of images
             if len(cube.data.shape) < 3:
-                self.log.warning('Input data are 2D; skipping outlier_detection')
+                self.log.warning('Input data are 2D; skipping outlier_detection_tso')
                 break
 
             self.log.info("Performing outlier detection on input images ...")
-            cube = self.outlier_detection.run(cube)
+            cube = self.outlier_detection_tso.run(cube)
 
             # Save crfints products
             # TODO: is any of this still necessary, or is it now handled correctly by the step?
             # or can it be replaced by invariant_filename?
-            if cube.meta.cal_step.outlier_detection == 'COMPLETE':
+            if cube.meta.cal_step.outlier_detection_tso == 'COMPLETE':
                 self.log.info("Saving crfints products with updated DQ arrays ...")
                 # preserve output filename
                 original_filename = cube.meta.filename

jwst/pipeline/tests/test_calwebb_image3.py

@@ -77,7 +77,7 @@ def test_run_image3_pipeline(make_dummy_association, in_memory):
             f"--logcfg={LOGCFG}",
             "--steps.tweakreg.skip=true",
             "--steps.skymatch.skip=true",
-            "--steps.outlier_detection.skip=true",
+            "--steps.outlier_detection_imaging.skip=true",
             "--steps.resample.skip=true",
             "--steps.source_catalog.skip=true",
             f"--in_memory={str(in_memory)}",]
@@ -96,7 +96,7 @@ def test_run_image3_single_file(make_dummy_cal_file):
             f"--logcfg={LOGCFG}",
             "--steps.tweakreg.skip=true",
             "--steps.skymatch.skip=true",
-            "--steps.outlier_detection.skip=true",
+            "--steps.outlier_detection_imaging.skip=true",
             "--steps.resample.skip=true",
             "--steps.source_catalog.skip=true",]
 

jwst/regtest/test_miri_lrs_slitless.py

@@ -82,8 +82,8 @@ def run_tso3_pipeline(run_tso_spec2_pipeline, rtdata_module):
     args = [
         "calwebb_tso3",
         ASN3_FILENAME,
-        "--steps.outlier_detection.save_results=true",
-        "--steps.outlier_detection.save_intermediate_results=true"
+        "--steps.outlier_detection_tso.save_results=true",
+        "--steps.outlier_detection_tso.save_intermediate_results=true"
     ]
     Step.from_cmdline(args)