JP-3719: Fix names for intermediate spectral outlier detection files (#…

…8735)

CHANGES.rst

@@ -61,14 +61,18 @@ outlier_detection
 - Fixed failures due to a missing ``wcs.array_shape`` attribute when the
   ``outlier_detection`` step was run standalone using e.g. ``strun`` [#8645]
 
+- Refactored separate modes into submodules instead of inheriting from a base class.
+  Moved non-JWST-specific code to stcal. [#8613]
+
+- Fixed file names and output directory for intermediate resampled spectral
+  images. Intermediate files now have suffix ``outlier_s2d`` and are saved to
+  the output directory alongside final products. [#8735]
+
 set_telescope_pointing
 ----------------------
 
 - replace usage of ``copy_arrays=True`` with ``memmap=False`` [#8660]
 
-- Refactored separate modes into submodules instead of inheriting from a base class.
-  Moved non-JWST-specific code to stcal. [#8613]
-
 resample_spec
 -------------
 

docs/jwst/outlier_detection/outlier_detection_spec.rst

@@ -29,14 +29,14 @@ Specifically, this routine performs the following operations (modified from the
 #. Resample all input images into a :py:class:`~jwst.datamodels.ModelContainer` using
    :py:class:`~jwst.resample.resample_spec.ResampleSpecData`
 
-   - Resampled images are written out to disk if the ``save_intermediate_results``
-     parameter is set to `True`
+   - Resampled images are written out to disk with suffix "outlier_s2d"
+     if the ``save_intermediate_results`` parameter is set to `True`.
    - **If resampling is turned off**, the original unrectified inputs are used to create
-     the median image for cosmic-ray detection
+     the median image for cosmic-ray detection.
 #. Create a median image from (possibly) resampled :py:class:`~jwst.datamodels.ModelContainer`
 
-   - The median image is written out to disk if the ``save_intermediate_results``
-     parameter is set to `True`
+   - The median image is written out to disk with suffix "median"
+     if the ``save_intermediate_results`` parameter is set to `True`
 #. Blot median image to match each original input image
 
    - **If resampling is turned off**, the median image is used for comparison

jwst/outlier_detection/imaging.py

@@ -60,8 +60,8 @@ def detect_outliers(
     if resample_data:
         # Start by creating resampled/mosaic images for
         # each group of exposures
-        output_path = make_output_path(basepath=input_models[0].meta.filename,
-                        suffix='')
+        output_path = make_output_path(
+            basepath=input_models[0].meta.filename, suffix='')
         output_path = os.path.dirname(output_path)
         resamp = resample.ResampleData(
             input_models,

jwst/outlier_detection/outlier_detection_step.py

@@ -193,7 +193,7 @@ def _guess_mode(self, input_models):
         if exptype in IFU_SPEC_MODES:
             return 'ifu'
 
-        self.log.error("Outlier detection failed for unknown/unsupported ",
+        self.log.error(f"Outlier detection failed for unknown/unsupported "
                        f"exposure type: {exptype}")
         return None
 

jwst/outlier_detection/spec.py

@@ -2,6 +2,7 @@
 Submodule for performing outlier detection on spectra.
 """
 import copy
+import os
 
 from stdatamodels.jwst import datamodels
 
@@ -56,8 +57,12 @@ def detect_outliers(
     if resample_data is True:
         # Start by creating resampled/mosaic images for
         #  each group of exposures
+        output_path = make_output_path(
+            basepath=input_models[0].meta.filename, suffix='')
+        output_path = os.path.dirname(output_path)
         resamp = resample_spec.ResampleSpecData(
             input_models,
+            output=output_path,
             single=True,
             blendheaders=False,
             wht_type=weight_type,
@@ -70,14 +75,6 @@ def detect_outliers(
         )
         median_wcs = resamp.output_wcs
         drizzled_models = resamp.do_drizzle(input_models)
-        if save_intermediate_results:
-            for model in drizzled_models:
-                model.meta.filename = make_output_path(
-                    basepath=model.meta.filename,
-                    suffix="_outlier_s2d.fits",
-                )
-                log.info("Writing out resampled spectra...")
-                model.save(model.meta.filename)
     else:
         drizzled_models = input_models
         for i in range(len(input_models)):

jwst/outlier_detection/tests/test_outlier_detection.py

@@ -6,6 +6,8 @@
 
 from stdatamodels.jwst import datamodels
 
+from jwst.assign_wcs import AssignWcsStep
+from jwst.assign_wcs.pointing import create_fitswcs
 from jwst.datamodels import ModelContainer
 from jwst.outlier_detection import OutlierDetectionStep
 from jwst.outlier_detection.utils import flag_resampled_model_crs
@@ -15,7 +17,7 @@
     TSO_IMAGE_MODES,
     CORON_IMAGE_MODES,
 )
-from jwst.assign_wcs.pointing import create_fitswcs
+from jwst.resample.tests.test_resample_step import miri_rate_model
 
 OUTLIER_DO_NOT_USE = np.bitwise_or(
     datamodels.dqflags.pixel["DO_NOT_USE"], datamodels.dqflags.pixel["OUTLIER"]
@@ -210,7 +212,7 @@ def test_outlier_step(we_three_sci, tmp_cwd):
     # Drop a CR on the science array
     container[0].data[12, 12] += 1
 
-    # Verify that intermediary files are removed
+    # Verify that intermediate files are removed
     OutlierDetectionStep.call(container)
     i2d_files = glob(os.path.join(tmp_cwd, '*i2d.fits'))
     median_files = glob(os.path.join(tmp_cwd, '*median.fits'))
@@ -232,13 +234,104 @@ def test_outlier_step(we_three_sci, tmp_cwd):
     # Verify CR is flagged
     assert result[0].dq[12, 12] == OUTLIER_DO_NOT_USE
 
-    # Verify that intermediary files are saved at the specified location
+    # Verify that intermediate files are saved at the specified location
     i2d_files = glob(os.path.join(tmp_cwd, '*i2d.fits'))
     median_files = glob(os.path.join(tmp_cwd, '*median.fits'))
     assert len(i2d_files) != 0
     assert len(median_files) != 0
 
 
+def test_outlier_step_spec(tmp_cwd, tmp_path):
+    """Test outlier step for spec data including saving intermediate results."""
+    output_dir = tmp_path / 'output'
+    output_dir.mkdir(exist_ok=True)
+    output_dir = str(output_dir)
+
+    # Make a MIRI model and assign a spectral wcs
+    miri_rate = miri_rate_model()
+    miri_cal = AssignWcsStep.call(miri_rate)
+
+    # Make it an exposure type outlier detection expects
+    miri_cal.meta.exposure.type = "MIR_LRS-FIXEDSLIT"
+
+    # Make a couple copies
+    container = ModelContainer([miri_cal, miri_cal.copy(), miri_cal.copy()])
+
+    # Give each image a unique name so output files don't overwrite
+    for i, model in enumerate(container):
+        model.meta.filename = f'test_{i}_cal.fits'
+
+    # Drop a CR on the science array in the first image
+    container[0].data[209, 37] += 1
+
+    # Verify that intermediate files are removed when not saved
+    # (s2d files are expected, i2d files are not, but we'll check
+    # for them to make sure the imaging extension didn't creep back in)
+    OutlierDetectionStep.call(container, output_dir=output_dir, save_results=True)
+    for dirname in [output_dir, tmp_cwd]:
+        result_files = glob(os.path.join(dirname, '*outlierdetectionstep.fits'))
+        i2d_files = glob(os.path.join(dirname, '*i2d*.fits'))
+        s2d_files = glob(os.path.join(dirname, '*outlier_s2d.fits'))
+        median_files = glob(os.path.join(dirname, '*median.fits'))
+
+        # intermediate files are removed
+        assert len(i2d_files) == 0
+        assert len(s2d_files) == 0
+        assert len(median_files) == 0
+
+        # result files are written to the output directory
+        if dirname == output_dir:
+            assert len(result_files) == len(container)
+        else:
+            assert len(result_files) == 0
+
+    # Call again, but save intermediate to the output path
+    result = OutlierDetectionStep.call(
+        container, save_results=True, save_intermediate_results=True,
+        output_dir=output_dir
+    )
+
+    # Make sure nothing changed in SCI array
+    for image, corrected in zip(container, result):
+        np.testing.assert_allclose(image.data, corrected.data)
+
+    # Verify CR is flagged
+    assert result[0].dq[209, 37] == OUTLIER_DO_NOT_USE
+
+    # Verify that intermediate files are saved at the specified location
+    for dirname in [output_dir, tmp_cwd]:
+        all_files = glob(os.path.join(dirname, '*.fits'))
+        result_files = glob(os.path.join(dirname, '*outlierdetectionstep.fits'))
+        i2d_files = glob(os.path.join(dirname, '*i2d*.fits'))
+        s2d_files = glob(os.path.join(dirname, '*outlier_s2d.fits'))
+        median_files = glob(os.path.join(dirname, '*median.fits'))
+        if dirname == output_dir:
+            # result files are written to the output directory
+            assert len(result_files) == len(container)
+
+            # s2d and median files are written to the output directory
+            assert len(s2d_files) == len(container)
+            assert len(median_files) == 1
+
+            # i2d files not written
+            assert len(i2d_files) == 0
+
+            # nothing else was written
+            assert len(all_files) == len(s2d_files) + len(median_files) + len(result_files)
+        else:
+            # nothing should be written to the current directory
+            assert len(result_files) == 0
+            assert len(s2d_files) == 0
+            assert len(median_files) == 0
+            assert len(i2d_files) == 0
+            assert len(all_files) == 0
+
+    miri_rate.close()
+    result.close()
+    for model in container:
+        model.close()
+
+
 def test_outlier_step_on_disk(we_three_sci, tmp_cwd):
     """Test whole step with an outlier including saving intermediate and results files"""
 

jwst/resample/resample.py

@@ -73,6 +73,7 @@ def __init__(self, input_models, output=None, single=False, blendheaders=True,
         if output is not None and '.fits' not in str(output):
             self.output_dir = output
             self.output_filename = None
+        self.intermediate_suffix = 'outlier_i2d'
 
         self.pscale_ratio = pscale_ratio
         self.single = single
@@ -276,16 +277,21 @@ def resample_many_to_many(self, input_models):
         """
         for exposure in input_models.models_grouped:
             output_model = self.blank_output
+
             # Determine output file type from input exposure filenames
             # Use this for defining the output filename
             indx = exposure[0].meta.filename.rfind('.')
             output_type = exposure[0].meta.filename[indx:]
             output_root = '_'.join(exposure[0].meta.filename.replace(
                 output_type, '').split('_')[:-1])
             if self.asn_id is not None:
-                output_model.meta.filename = f'{output_root}_{self.asn_id}_outlier_i2d{output_type}'
+                output_model.meta.filename = (
+                    f'{output_root}_{self.asn_id}_'
+                    f'{self.intermediate_suffix}{output_type}')
             else:
-                output_model.meta.filename = f'{output_root}_outlier_i2d{output_type}'
+                output_model.meta.filename = (
+                    f'{output_root}_'
+                    f'{self.intermediate_suffix}{output_type}')
 
             # Initialize the output with the wcs
             driz = gwcs_drizzle.GWCSDrizzle(output_model, pixfrac=self.pixfrac,

jwst/resample/resample_spec.py

@@ -66,6 +66,7 @@ def __init__(self, input_models, output=None, single=False, blendheaders=False,
         if output is not None and '.fits' not in str(output):
             self.output_dir = output
             self.output_filename = None
+        self.intermediate_suffix = 'outlier_s2d'
 
         self.pscale_ratio = pscale_ratio
         self.single = single
@@ -78,7 +79,7 @@ def __init__(self, input_models, output=None, single=False, blendheaders=False,
         self.in_memory = kwargs.get('in_memory', True)
         self._recalc_pscale_ratio = False
 
-        log.info(f"Driz parameter kernal: {self.kernel}")
+        log.info(f"Driz parameter kernel: {self.kernel}")
         log.info(f"Driz parameter pixfrac: {self.pixfrac}")
         log.info(f"Driz parameter fillval: {self.fillval}")
         log.info(f"Driz parameter weight_type: {self.weight_type}")

jwst/resample/tests/test_resample_step.py

@@ -37,8 +37,7 @@ def _set_photom_kwd(im):
         )
 
 
-@pytest.fixture
-def miri_rate():
+def miri_rate_model():
     xsize = 72
     ysize = 416
     shape = (ysize, xsize)
@@ -87,6 +86,11 @@ def miri_rate():
         'start_time': 58119.8333,
         'type': 'MIR_LRS-SLITLESS',
         'zero_frame': False}
+    return im
+
+@pytest.fixture
+def miri_rate():
+    im = miri_rate_model()
     yield im
     im.close()