From 9f1bbbd67db92ea588453911be3c0b116de56cf9 Mon Sep 17 00:00:00 2001 From: Nadia Dencheva Date: Thu, 19 Dec 2024 15:46:46 -0500 Subject: [PATCH 1/5] AL-852: GWCS inverse transform should respect its bounding box (#8554) Co-authored-by: Ned Molter Co-authored-by: Ned Molter Co-authored-by: Melanie Clarke Co-authored-by: Tyler Pauly --- changes/8554.assign_wcs.rst | 2 + changes/8554.resample.rst | 1 + changes/8554.skymatch.rst | 1 + jwst/assign_wcs/miri.py | 2 + jwst/assign_wcs/util.py | 2 +- .../tests/test_outlier_detection.py | 50 +++++++++++-------- jwst/resample/resample_utils.py | 9 +++- jwst/resample/tests/test_resample_step.py | 7 ++- jwst/resample/tests/test_utils.py | 4 +- jwst/skymatch/skyimage.py | 2 +- jwst/tweakreg/tests/test_multichip_jwst.py | 6 +-- pyproject.toml | 14 +++--- 12 files changed, 63 insertions(+), 37 deletions(-) create mode 100644 changes/8554.assign_wcs.rst create mode 100644 changes/8554.resample.rst create mode 100644 changes/8554.skymatch.rst diff --git a/changes/8554.assign_wcs.rst b/changes/8554.assign_wcs.rst new file mode 100644 index 0000000000..c36dd9d167 --- /dev/null +++ b/changes/8554.assign_wcs.rst @@ -0,0 +1,2 @@ +Use the range of points in the TabularModel to adjust the bounding_box in a MIRI LRS FIXEDSLIT observation. +Ignore the bounding_box in running the inverse WCS transform when computing crpix. diff --git a/changes/8554.resample.rst b/changes/8554.resample.rst new file mode 100644 index 0000000000..bda0397ae8 --- /dev/null +++ b/changes/8554.resample.rst @@ -0,0 +1 @@ +Ignore the bounding_box in the inverse WCS transform in reproject. diff --git a/changes/8554.skymatch.rst b/changes/8554.skymatch.rst new file mode 100644 index 0000000000..1561af36b6 --- /dev/null +++ b/changes/8554.skymatch.rst @@ -0,0 +1 @@ +Ignore the bounding_box of an observation when computing sky statistics. diff --git a/jwst/assign_wcs/miri.py b/jwst/assign_wcs/miri.py index 59ec01d959..f15938ce88 100644 --- a/jwst/assign_wcs/miri.py +++ b/jwst/assign_wcs/miri.py @@ -310,6 +310,8 @@ def lrs_xytoabl(input_model, reference_files): dxmodel = models.Tabular1D(lookup_table=xshiftref, points=ycen_subarray, name='xshiftref', bounds_error=False, fill_value=np.nan) + if input_model.meta.exposure.type.lower() == 'mir_lrs-fixedslit': + bb_sub = (bb_sub[0], (dxmodel.points[0].min(), dxmodel.points[0].max())) # Fit for the wavelength as a function of Y # Reverse the vectors so that yinv is increasing (needed for spline fitting function) # Spline fit with enforced smoothness diff --git a/jwst/assign_wcs/util.py b/jwst/assign_wcs/util.py index a577e8a8e4..3735a45bee 100644 --- a/jwst/assign_wcs/util.py +++ b/jwst/assign_wcs/util.py @@ -127,7 +127,7 @@ def compute_scale(wcs: WCS, fiducial: Union[tuple, np.ndarray], if spectral and disp_axis is None: raise ValueError('If input WCS is spectral, a disp_axis must be given') - crpix = np.array(wcs.invert(*fiducial)) + crpix = np.array(wcs.invert(*fiducial, with_bounding_box=False)) delta = np.zeros_like(crpix) spatial_idx = np.where(np.array(wcs.output_frame.axes_type) == 'SPATIAL')[0] diff --git a/jwst/outlier_detection/tests/test_outlier_detection.py b/jwst/outlier_detection/tests/test_outlier_detection.py index 008ae19c64..7e03300422 100644 --- a/jwst/outlier_detection/tests/test_outlier_detection.py +++ b/jwst/outlier_detection/tests/test_outlier_detection.py @@ -6,11 +6,9 @@ from gwcs.wcs import WCS from stdatamodels.jwst import datamodels -from stcal.alignment.util import compute_s_region_imaging from jwst.datamodels import ModelContainer, ModelLibrary from jwst.assign_wcs import AssignWcsStep -from jwst.assign_wcs.pointing import create_fitswcs from jwst.outlier_detection import OutlierDetectionStep from jwst.outlier_detection.utils import _flag_resampled_model_crs from jwst.outlier_detection.outlier_detection_step import ( @@ -108,21 +106,17 @@ def test_flag_cr(sci_blot_image_pair): assert sci.dq[10, 10] == datamodels.dqflags.pixel["GOOD"] -# not a fixture - now has options -def we_many_sci( - numsci=3, sigma=0.02, background=1.5, signal=7, exptype="MIR_IMAGE", tsovisit=False -): - """Provide numsci science images with different noise but identical source - and same background level""" - shape = (21, 20) +def make_sci1(shape): + """Needs to be a fixture because we want to change exposure.type + in the subsequent tests without rerunning AssignWCS""" sci1 = datamodels.ImageModel(shape) # Populate keywords sci1.meta.instrument.name = "MIRI" sci1.meta.instrument.detector = "MIRIMAGE" - sci1.meta.exposure.type = exptype - sci1.meta.visit.tsovisit = tsovisit + sci1.meta.exposure.type = "MIR_IMAGE" + sci1.meta.visit.tsovisit = False sci1.meta.observation.date = "2020-01-01" sci1.meta.observation.time = "00:00:00" sci1.meta.telescope = "JWST" @@ -135,6 +129,8 @@ def we_many_sci( sci1.meta.wcsinfo.roll_ref = 0 sci1.meta.wcsinfo.ra_ref = 1.5e-5 sci1.meta.wcsinfo.dec_ref = 1.5e-5 + sci1.meta.wcsinfo.v2_ref = 0 + sci1.meta.wcsinfo.v3_ref = 0 sci1.meta.wcsinfo.v3yangle = 0 sci1.meta.wcsinfo.vparity = -1 sci1.meta.wcsinfo.pc1_1 = 1 @@ -148,11 +144,29 @@ def we_many_sci( sci1.meta.wcsinfo.cunit1 = "deg" sci1.meta.wcsinfo.cunit2 = "deg" sci1.meta.background.subtracted = False - sci1.meta.background.level = background + sci1.meta.background.level = 1.5 + + sci1 = AssignWcsStep.call(sci1) + + sci1.meta.filename = "foo1_cal.fits" + + # add pixel areas + sci1.meta.photometry.pixelarea_steradians = 1.0 + sci1.meta.photometry.pixelarea_arcsecsq = 1.0 + return sci1 + + +# not a fixture - now has options +def we_many_sci( + numsci=3, sigma=0.02, background=1.5, signal=7, exptype="MIR_IMAGE", tsovisit=False +): + """Provide numsci science images with different noise but identical source + and same background level""" + shape = (21,20) + sci1 = make_sci1(shape) + sci1.meta.exposure.type = exptype + sci1.meta.visit.tsovisit = tsovisit - # Replace the FITS-type WCS with an Identity WCS - sci1.meta.wcs = create_fitswcs(sci1) - sci1.meta.wcsinfo.s_region = compute_s_region_imaging(sci1.meta.wcs, shape=shape, center=False) rng = np.random.default_rng(720) sci1.data = rng.normal(loc=background, size=shape, scale=sigma) sci1.err = np.zeros(shape) + sigma @@ -160,11 +174,7 @@ def we_many_sci( # update the noise for this source to include the photon/measurement noise sci1.err[7, 7] = np.sqrt(sigma ** 2 + signal) sci1.var_rnoise = np.zeros(shape) + 1.0 - sci1.meta.filename = "foo1_cal.fits" - # add pixel areas - sci1.meta.photometry.pixelarea_steradians = 1.0 - sci1.meta.photometry.pixelarea_arcsecsq = 1.0 # Make copies with different noise all_sci = [sci1] @@ -650,7 +660,7 @@ def test_drizzle_and_median_with_resample(three_sci_as_asn, tmp_cwd): 0.7) assert isinstance(wcs, WCS) - assert median.shape == (21,20) + assert median.shape == (34,34) resamp.single = False with pytest.raises(ValueError): diff --git a/jwst/resample/resample_utils.py b/jwst/resample/resample_utils.py index 4d126ed28c..da917d7e41 100644 --- a/jwst/resample/resample_utils.py +++ b/jwst/resample/resample_utils.py @@ -163,11 +163,18 @@ def reproject(wcs1, wcs2): """ try: + # Here we want to use the WCS API functions so that a Sliced WCS + # will work as well. However, the API functions do not accept + # keyword arguments and `with_bounding_box=False` cannot be passsed. + # We delete the bounding box on a copy of the WCS - yes, inefficient. forward_transform = wcs1.pixel_to_world_values - backward_transform = wcs2.world_to_pixel_values + wcs_no_bbox = deepcopy(wcs2) + wcs_no_bbox.bounding_box = None + backward_transform = wcs_no_bbox.world_to_pixel_values except AttributeError as err: raise TypeError("Input should be a WCS") from err + def _reproject(x, y): sky = forward_transform(x, y) flat_sky = [] diff --git a/jwst/resample/tests/test_resample_step.py b/jwst/resample/tests/test_resample_step.py index 75eddb8530..c9eb4ec1b5 100644 --- a/jwst/resample/tests/test_resample_step.py +++ b/jwst/resample/tests/test_resample_step.py @@ -831,8 +831,8 @@ def test_resample_undefined_variance(nircam_rate, shape): @pytest.mark.parametrize('ratio', [0.7, 1.2]) @pytest.mark.parametrize('rotation', [0, 15, 135]) -@pytest.mark.parametrize('crpix', [(256, 488), (700, 124)]) -@pytest.mark.parametrize('crval', [(50, 77), (20, -30)]) +@pytest.mark.parametrize('crpix', [(600, 550), (601, 551)]) +@pytest.mark.parametrize('crval', [(22.04, 11.98), (22.041, 11.981)]) @pytest.mark.parametrize('shape', [(1205, 1100)]) def test_custom_wcs_resample_imaging(nircam_rate, ratio, rotation, crpix, crval, shape): im = AssignWcsStep.call(nircam_rate, sip_approx=False) @@ -848,6 +848,9 @@ def test_custom_wcs_resample_imaging(nircam_rate, ratio, rotation, crpix, crval, t = result.meta.wcs.forward_transform + # make sure results are nontrivial + assert not np.all(np.isnan(result.data)) + # test rotation pc = t['pc_rotation_matrix'].matrix.value orientation = np.rad2deg(np.arctan2(pc[0, 1], pc[1, 1])) diff --git a/jwst/resample/tests/test_utils.py b/jwst/resample/tests/test_utils.py index 7e7deb3070..518fc61998 100644 --- a/jwst/resample/tests/test_utils.py +++ b/jwst/resample/tests/test_utils.py @@ -38,7 +38,7 @@ def wcs_gwcs(): crpix = (500.0, 500.0) shape = (1000, 1000) pscale = 0.06 / 3600 - + prj = astmodels.Pix2Sky_TAN() fiducial = np.array(crval) @@ -193,7 +193,7 @@ def test_reproject(wcs1, wcs2, offset, request): wcs1 = request.getfixturevalue(wcs1) wcs2 = request.getfixturevalue(wcs2) x = np.arange(150, 200) - + f = reproject(wcs1, wcs2) res = f(x, x) assert_allclose(x, res[0] + offset) diff --git a/jwst/skymatch/skyimage.py b/jwst/skymatch/skyimage.py index 65bc943bec..8a6ca6f3c8 100644 --- a/jwst/skymatch/skyimage.py +++ b/jwst/skymatch/skyimage.py @@ -619,7 +619,7 @@ def calc_sky(self, overlap=None, delta=True): continue # set pixels in 'fill_mask' that are inside a polygon to True: - x, y = self.wcs_inv(ra, dec) + x, y = self.wcs_inv(ra, dec, with_bounding_box=False) poly_vert = list(zip(*[x, y])) polygon = region.Polygon(True, poly_vert) diff --git a/jwst/tweakreg/tests/test_multichip_jwst.py b/jwst/tweakreg/tests/test_multichip_jwst.py index 4402f7b897..352d089e5a 100644 --- a/jwst/tweakreg/tests/test_multichip_jwst.py +++ b/jwst/tweakreg/tests/test_multichip_jwst.py @@ -81,7 +81,7 @@ def _make_gwcs_wcs(fits_hdr): Mapping((1, 2), name='xtyt')) c2tan.name = 'Cartesian 3D to TAN' - tan2c = (Mapping((0, 0, 1), n_inputs=2, name='xtyt2xyz') | + tan2c = (Mapping((0, 0, 1), name='xtyt2xyz') | (Const1D(1, name='one') & Identity(2, name='I(2D)'))) tan2c.name = 'TAN to cartesian 3D' @@ -376,7 +376,7 @@ def test_multichip_alignment_step_rel(monkeypatch): format='ascii.ecsv', delimiter=' ', names=['RA', 'DEC'] ) - x, y = wr.world_to_pixel(refcat['RA'], refcat['DEC']) + x, y = wr.invert(refcat['RA'].value, refcat['DEC'].value, with_bounding_box=False) refcat['x'] = x refcat['y'] = y mr.tweakreg_catalog = refcat @@ -459,7 +459,7 @@ def test_multichip_alignment_step_abs(monkeypatch): format='ascii.ecsv', delimiter=' ', names=['RA', 'DEC'] ) - x, y = wr.world_to_pixel(refcat['RA'], refcat['DEC']) + x, y = wr.invert(refcat['RA'].value, refcat['DEC'].value, with_bounding_box=False) refcat['x'] = x refcat['y'] = y mr.tweakreg_catalog = refcat diff --git a/pyproject.toml b/pyproject.toml index a944d7ba44..2d7907ed2d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,28 +18,28 @@ classifiers = [ "Programming Language :: Python :: 3.12", ] dependencies = [ - "asdf>=3.1.0,<5", - "astropy>=5.3", + "asdf>=3.3,<5", + "astropy>=6.1", "BayesicFitting>=3.0.1", "crds>=12.0.3", "drizzle>=2.0.0", - "gwcs>=0.21.0,<0.23.0", - "numpy>=1.22,<2.0", + "gwcs>=0.22.0,<0.23.0", + "numpy>=1.24,<2.0", "opencv-python-headless>=4.6.0.66", "photutils>=1.5.0", "poppy>=1.0.2", "pyparsing>=2.2.1", "requests>=2.22", "scikit-image>=0.19", - "scipy>=1.9.3", + "scipy>=1.14.1", "spherical-geometry>=1.2.22", - "stcal @ git+https://github.com/spacetelescope/stcal.git@30dd76949b6c46ed1ea2a410864252ce6f120c89", + "stcal @ git+https://github.com/spacetelescope/stcal.git@main", "stdatamodels @ git+https://github.com/spacetelescope/stdatamodels.git@main", "stpipe @ git+https://github.com/spacetelescope/stpipe.git@main", "stsci.imagestats>=1.6.3", "synphot>=1.2", "tweakwcs>=0.8.8", - "asdf-astropy>=0.3.0", + "asdf-astropy>=0.5.0", "wiimatch>=0.2.1", "packaging>20.0", "importlib-metadata>=4.11.4", From 2968d3f144f945a483f8c92fc67e4c52d11a91df Mon Sep 17 00:00:00 2001 From: Brett Graham Date: Fri, 20 Dec 2024 09:51:43 -0500 Subject: [PATCH 2/5] update minimum deps (#9021) --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 2d7907ed2d..12c5996d16 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,13 +24,13 @@ dependencies = [ "crds>=12.0.3", "drizzle>=2.0.0", "gwcs>=0.22.0,<0.23.0", - "numpy>=1.24,<2.0", + "numpy>=1.25,<2.0", "opencv-python-headless>=4.6.0.66", "photutils>=1.5.0", "poppy>=1.0.2", "pyparsing>=2.2.1", "requests>=2.22", - "scikit-image>=0.19", + "scikit-image>=0.20.0", "scipy>=1.14.1", "spherical-geometry>=1.2.22", "stcal @ git+https://github.com/spacetelescope/stcal.git@main", From dfd1fc7f2f1a61028e623e0f29e6894e530df2cd Mon Sep 17 00:00:00 2001 From: Maria Date: Fri, 20 Dec 2024 15:51:31 -0500 Subject: [PATCH 3/5] JP-3622 Update refpix step for NIR detectors to use convolution kernel (#8726) Co-authored-by: David Law Co-authored-by: Melanie Clarke Co-authored-by: Tyler Pauly --- changes/8726.refpix.rst | 1 + docs/jwst/refpix/arguments.rst | 22 +++ docs/jwst/refpix/description.rst | 12 ++ jwst/refpix/optimized_convolution.py | 180 ++++++++++++++++++ jwst/refpix/reference_pixels.py | 136 +++++++++---- jwst/refpix/refpix_step.py | 35 +++- .../tests/test_optimized_convolution.py | 65 +++++++ jwst/refpix/tests/test_refpix.py | 76 +++++--- jwst/regtest/test_nircam_image.py | 32 ++++ 9 files changed, 493 insertions(+), 66 deletions(-) create mode 100644 changes/8726.refpix.rst create mode 100644 jwst/refpix/optimized_convolution.py create mode 100644 jwst/refpix/tests/test_optimized_convolution.py diff --git a/changes/8726.refpix.rst b/changes/8726.refpix.rst new file mode 100644 index 0000000000..0639bf5208 --- /dev/null +++ b/changes/8726.refpix.rst @@ -0,0 +1 @@ +Implemented SIRS algorithm instead of running median for side pixels of NIR full-frame data. Running median is still default. diff --git a/docs/jwst/refpix/arguments.rst b/docs/jwst/refpix/arguments.rst index 59126c2db4..a5c92404e9 100644 --- a/docs/jwst/refpix/arguments.rst +++ b/docs/jwst/refpix/arguments.rst @@ -52,3 +52,25 @@ in IRS2 mode will be processed along with the normal pixels and preserved in the output. This option is intended for calibration or diagnostic reductions only. For normal science operation, this argument should always be False, so that interleaved pixels are stripped before continuing processing. + +* ``--refpix_algorithm`` + +The ``refpix_algorithm`` argument is only relevant for all NIR full-frame +data, and can be set to 'median' (default) to use the running median or +'sirs' to use the Simple Improved Reference Subtraction (SIRS). + +* ``--sigreject`` + +The ``sigreject`` argument is the number of sigmas to reject as outliers in the +SIRS algorithm. The value is expected to be a float. + +* ``--gaussmooth`` + +The ``gaussmooth`` argument is the width of Gaussian smoothing kernel to use as +a low-pass filter. The numerical value is expected to be a float. + +* ``--halfwidth`` + +The ``halfwidth`` argument is the half-width of convolution kernel to build. The +numerical value is expected to be an integer. + diff --git a/docs/jwst/refpix/description.rst b/docs/jwst/refpix/description.rst index f5fb2d8701..0dd86b0177 100644 --- a/docs/jwst/refpix/description.rst +++ b/docs/jwst/refpix/description.rst @@ -76,6 +76,18 @@ NIR Detector Data (set by the step parameter ``side_gain``, which defaults to 1.0) is subtracted from the full group on a row-by-row basis. Note that the ``odd_even_rows`` parameter is ignored for NIR data when the side reference pixels are processed. + If the ``--refpix_algorithm`` option is set to 'sirs', the Simple Improved + Reference Subtraction (SIRS) method will be used instead of the running median. + The SIRS revision uses the left and right side reference pixels as described + in https://doi.org/10.1117/1.JATIS.8.2.028002. This implementation uses a + mathematically equivalent formulation using convolution kernels rather than + Fourier transforms, with the convolution kernel truncated where the weights + approach zero. There are two convolution kernels for each readout channel, + one for each of the left and right reference pixels. These kernels are the + Fourier transforms of the weight coefficients (further description in the paper). + The approach implemented here makes nearly optimal use of the reference pixels. + It reduces read noise by 5-10% relative to a running median filter of the + reference pixels, and reduces 1/f "striping" noise by a larger factor than this. #. Transform the data back to the JWST focal plane, or DMS, frame. MIR Detector Data diff --git a/jwst/refpix/optimized_convolution.py b/jwst/refpix/optimized_convolution.py new file mode 100644 index 0000000000..02872114b7 --- /dev/null +++ b/jwst/refpix/optimized_convolution.py @@ -0,0 +1,180 @@ +# +# Module for using the Simple Improved Reference Subtraction (SIRS) algorithm +# to improve the 1/f noise, only for full frame non-IRS2 NIR data +# + +import logging +import numpy as np + +log = logging.getLogger(__name__) +log.setLevel(logging.DEBUG) + + +def make_kernels(sirs_kernel_model, detector, gaussmooth, halfwidth): + """ + Make convolution kernels from Fourier coefficients in the reference file. + + Parameters: + ----------- + + sirs_kernel_model : `~jwst.datamodels.SIRSKernelModel` + Data model containing the Fourier coefficients from the reference files for + Simple Improved Reference Subtraction (SIRS) + + detector : str + Name of the detector of the input data + + gaussmooth : float + Width of Gaussian smoothing kernel to use as a low-pass filter on reference file's coefficients + + halfwidth : int + Half-width of convolution kernel to build from reference file's coefficients + + Returns: + -------- + kernels: list + List of kernels appropriate for convolution with the left and right reference pixels. + + """ + + gamma, zeta = get_conv_kernel_coeffs(sirs_kernel_model, detector) + if gamma is None or zeta is None: + log.info(f'Optimized convolution kernel coefficients NOT found for detector {detector}') + return None + + kernels_left = [] + kernels_right = [] + for chan in range(gamma.shape[0]): + n = len(gamma[chan]) - 1 + kernel_left = np.fft.fftshift(np.fft.irfft(gamma[chan]))[n - halfwidth:n + halfwidth + 1] + kernel_right = np.fft.fftshift(np.fft.irfft(zeta[chan]))[n - halfwidth:n + halfwidth + 1] + + x = np.arange(-halfwidth, halfwidth + 1) + window = np.exp(-x ** 2 / (2 * gaussmooth ** 2)) + window /= np.sum(window) + + kernel_right = np.convolve(kernel_right, window, mode='same') + kernel_left = np.convolve(kernel_left, window, mode='same') + + kernels_right += [kernel_right] + kernels_left += [kernel_left] + + return [kernels_left, kernels_right] + + +def get_conv_kernel_coeffs(sirs_kernel_model, detector): + """ + Get the convolution kernels coefficients from the reference file + + Parameters: + ----------- + + sirs_kernel_model : `~jwst.datamodels.SIRSKernelModel` + Data model containing the Fourier coefficients from the reference files for + Simple Improved Reference Subtraction (SIRS) + + detector : str + Name of the detector of the input data + + Returns: + -------- + + gamma: numpy array + Fourier coefficients + + zeta: numpy array + Fourier coefficients + """ + mdl_dict = sirs_kernel_model.to_flat_dict() + gamma, zeta = None, None + for item in mdl_dict: + det = item.split(sep='.')[0] + if detector.lower() == det.lower(): + arr_name = item.split(sep='.')[1] + if arr_name == 'gamma': + gamma = np.array(mdl_dict[item]) + elif arr_name == 'zeta': + zeta = np.array(mdl_dict[item]) + if gamma is not None and zeta is not None: + break + return gamma, zeta + + +def apply_conv_kernel(data, kernels, sigreject=4.0): + """ + Apply the convolution kernel. + + Parameters: + ----------- + + data : 2-D numpy array + Data to be corrected + + kernels : list + List containing the left and right kernels + + sigreject: float + Number of sigmas to reject as outliers + + Returns: + -------- + + data : 2-D numpy array + Data model with convolution + """ + data = data.astype(float) + npix = data.shape[-1] + + kernels_l, kernels_r = kernels + nchan = len(kernels_l) + + L = data[:, :4] + R = data[:, -4:] + + # Find the approximate standard deviations of the reference pixels + # using an outlier-robust median approach. Mask pixels that differ + # by more than sigreject sigma from this level. + # NOTE: The Median Absolute Deviation (MAD) is calculated as the + # median of the absolute differences between data values and their + # median. For normal distribution MAD is equal to 1.48 times the + # standard deviation but is a more robust estimate of the dispersion + # of data values.The calculation of MAD is straightforward but + # time-consuming, especially if MAD estimates are needed for the + # local environment around every pixel of a large image. The + # calculation is MAD = np.median(np.abs(x-np.median(x))). + # Reference: https://www.interstellarmedium.org/numerical_tools/mad/ + MAD = 1.48 + medL = np.median(L) + sigL = MAD * np.median(np.abs(L - medL)) + medR = np.median(R) + sigR = MAD * np.median(np.abs(R - medR)) + + # nL and nR are the number of good reference pixels in the left and right + # channel in each row. These will be used in lieu of replacing the values + # of those pixels directly. + goodL = 1 * (np.abs(L - medL) <= sigreject * sigL) + nL = np.sum(goodL, axis=1) + goodR = 1 * (np.abs(R - medR) <= sigreject * sigR) + nR = np.sum(goodR, axis=1) + + # Average of the left and right channels, replacing masked pixels with zeros. + # Appropriate normalization factors will be computed later. + L = np.sum(L * goodL, axis=1) / 4 + R = np.sum(R * goodR, axis=1) / 4 + for chan in range(nchan): + kernel_l = kernels_l[chan] + kernel_r = kernels_r[chan] + + # Compute normalizations so that we don't have to directly + # replace the values of flagged/masked reference pixels. + normL = np.convolve(np.ones(nL.shape), kernel_l, mode='same') + normL /= np.convolve(nL / 4, kernel_l, mode='same') + normR = np.convolve(np.ones(nR.shape), kernel_r, mode='same') + normR /= np.convolve(nR / 4, kernel_r, mode='same') + template = np.convolve(L, kernel_l, mode='same') * normL + template += np.convolve(R, kernel_r, mode='same') * normR + data[:, chan * npix * 3 // 4:(chan + 1) * npix * 3 // 4] -= template[:, np.newaxis] + + log.debug('Optimized convolution kernel applied') + return data + diff --git a/jwst/refpix/reference_pixels.py b/jwst/refpix/reference_pixels.py index 734a1e33bd..7353af451a 100644 --- a/jwst/refpix/reference_pixels.py +++ b/jwst/refpix/reference_pixels.py @@ -51,6 +51,7 @@ from ..lib import pipe_utils, reffile_utils from .irs2_subtract_reference import make_irs2_mask +from .optimized_convolution import make_kernels, apply_conv_kernel log = logging.getLogger(__name__) log.setLevel(logging.DEBUG) @@ -134,7 +135,7 @@ SUBARRAY_SKIPPED = 3 -class Dataset(): +class Dataset: """Base Class to handle passing stuff from routine to routine Parameters: @@ -167,6 +168,10 @@ class Dataset(): flag that controls whether odd and even-numbered rows are handled separately (MIR only) + conv_kernel_params : dict + Dictionary containing the parameters needed for the optimized convolution kernel + for Simple Improved Reference Subtraction (SIRS) + """ def __init__(self, input_model, @@ -174,8 +179,15 @@ def __init__(self, input_model, use_side_ref_pixels, side_smoothing_length, side_gain, + conv_kernel_params, odd_even_rows): + self.refpix_algorithm = conv_kernel_params['refpix_algorithm'] + self.sirs_kernel_model = conv_kernel_params['sirs_kernel_model'] + self.sigreject = conv_kernel_params['sigreject'] + self.gaussmooth = conv_kernel_params['gaussmooth'] + self.halfwidth = conv_kernel_params['halfwidth'] + if (input_model.meta.subarray.xstart is None or input_model.meta.subarray.ystart is None or input_model.meta.subarray.xsize is None or @@ -373,12 +385,17 @@ def log_parameters(self): if not self.is_subarray: log.info('NIR full frame data') log.info('The following parameters are valid for this mode:') - log.info(f'use_side_ref_pixels = {self.use_side_ref_pixels}') - log.info(f'odd_even_columns = {self.odd_even_columns}') - log.info(f'side_smoothing_length = {self.side_smoothing_length}') - log.info(f'side_gain = {self.side_gain}') - log.info('The following parameter is not applicable and is ignored:') - log.info(f'odd_even_rows = {self.odd_even_rows}') + if self.refpix_algorithm == 'median': + log.info(f'use_side_ref_pixels = {self.use_side_ref_pixels}') + log.info(f'odd_even_columns = {self.odd_even_columns}') + log.info(f'side_smoothing_length = {self.side_smoothing_length}') + log.info(f'side_gain = {self.side_gain}') + log.info('The following parameter is not applicable and is ignored:') + log.info(f'odd_even_rows = {self.odd_even_rows}') + elif self.refpix_algorithm == 'sirs': + log.info(f'sigreject = {self.sigreject}') + log.info(f'gaussmooth = {self.gaussmooth}') + log.info(f'halfwidth = {self.halfwidth}') else: log.info('NIR subarray data') # Transform the pixeldq array from DMS to detector coords @@ -491,19 +508,24 @@ class NIRDataset(Dataset): side_gain: float gain to use in applying the side reference pixel correction + conv_kernel_params : dict + Dictionary containing the parameters needed for the optimized convolution kernel + """ def __init__(self, input_model, odd_even_columns, use_side_ref_pixels, side_smoothing_length, - side_gain): + side_gain, + conv_kernel_params): super(NIRDataset, self).__init__(input_model, odd_even_columns, use_side_ref_pixels, side_smoothing_length, side_gain, + conv_kernel_params, odd_even_rows=False) # Set appropriate NIR sections @@ -1149,10 +1171,28 @@ def do_side_correction(self, group): """ - left = self.calculate_side_ref_signal(group, 0, 3) - right = self.calculate_side_ref_signal(group, 2044, 2047) - sidegroup = self.combine_ref_signals(left, right) - corrected_group = self.apply_side_correction(group, sidegroup) + continue_apply_conv_kernel = False + # Check if convolution kernels for this detector are in the reference file + # and if not, proceed with side-pixel correction as usual + if self.refpix_algorithm == 'sirs' and self.sirs_kernel_model is not None: + kernels = make_kernels(self.sirs_kernel_model, + self.input_model.meta.instrument.detector, + self.gaussmooth, + self.halfwidth) + if kernels is None: + log.info('The REFPIX step will use the running median') + else: + continue_apply_conv_kernel = True + # + # Apply optimized convolution kernel + if continue_apply_conv_kernel: + corrected_group = apply_conv_kernel(group, kernels, sigreject=self.sigreject) + else: + # use running median + left = self.calculate_side_ref_signal(group, 0, 3) + right = self.calculate_side_ref_signal(group, 2044, 2047) + sidegroup = self.combine_ref_signals(left, right) + corrected_group = self.apply_side_correction(group, sidegroup) return corrected_group def do_corrections(self): @@ -1168,6 +1208,7 @@ def do_fullframe_corrections(self): """Do Reference Pixels Corrections for all amplifiers, NIR detectors First read of each integration is NOT subtracted, as the signal is removed in the superbias subtraction step""" + # # First transform pixeldq array to detector coordinates self.DMS_to_detector_dq() @@ -1578,13 +1619,15 @@ class MIRIDataset(Dataset): """ def __init__(self, input_model, - odd_even_rows): + odd_even_rows, + conv_kernel_params): super(MIRIDataset, self).__init__(input_model, odd_even_columns=False, use_side_ref_pixels=False, side_smoothing_length=False, side_gain=False, + conv_kernel_params=conv_kernel_params, odd_even_rows=odd_even_rows) self.reference_sections = MIR_reference_sections @@ -1923,7 +1966,8 @@ def create_dataset(input_model, use_side_ref_pixels, side_smoothing_length, side_gain, - odd_even_rows): + odd_even_rows, + conv_kernel_params): """Create a dataset object from an input model. Parameters: @@ -1952,6 +1996,8 @@ def create_dataset(input_model, flag that controls whether odd and even-numbered rows are handled separately (MIR only) + conv_kernel_params : dict + Dictionary containing the parameters needed for the optimized convolution kernel """ detector = input_model.meta.instrument.detector @@ -1966,110 +2012,128 @@ def create_dataset(input_model, if detector[:3] == 'MIR': return MIRIDataset(input_model, - odd_even_rows) + odd_even_rows, + conv_kernel_params) elif detector == 'NRS1': return NRS1Dataset(input_model, odd_even_columns, use_side_ref_pixels, side_smoothing_length, - side_gain) + side_gain, + conv_kernel_params) elif detector == 'NRS2': return NRS2Dataset(input_model, odd_even_columns, use_side_ref_pixels, side_smoothing_length, - side_gain) + side_gain, + conv_kernel_params) elif detector == 'NRCA1': return NRCA1Dataset(input_model, odd_even_columns, use_side_ref_pixels, side_smoothing_length, - side_gain) + side_gain, + conv_kernel_params) elif detector == 'NRCA2': return NRCA2Dataset(input_model, odd_even_columns, use_side_ref_pixels, side_smoothing_length, - side_gain) + side_gain, + conv_kernel_params) elif detector == 'NRCA3': return NRCA3Dataset(input_model, odd_even_columns, use_side_ref_pixels, side_smoothing_length, - side_gain) + side_gain, + conv_kernel_params) elif detector == 'NRCA4': return NRCA4Dataset(input_model, odd_even_columns, use_side_ref_pixels, side_smoothing_length, - side_gain) + side_gain, + conv_kernel_params) elif detector == 'NRCALONG': return NRCALONGDataset(input_model, odd_even_columns, use_side_ref_pixels, side_smoothing_length, - side_gain) + side_gain, + conv_kernel_params) elif detector == 'NRCB1': return NRCB1Dataset(input_model, odd_even_columns, use_side_ref_pixels, side_smoothing_length, - side_gain) + side_gain, + conv_kernel_params) elif detector == 'NRCB2': return NRCB2Dataset(input_model, odd_even_columns, use_side_ref_pixels, side_smoothing_length, - side_gain) + side_gain, + conv_kernel_params) elif detector == 'NRCB3': return NRCB3Dataset(input_model, odd_even_columns, use_side_ref_pixels, side_smoothing_length, - side_gain) + side_gain, + conv_kernel_params) elif detector == 'NRCB4': return NRCB4Dataset(input_model, odd_even_columns, use_side_ref_pixels, side_smoothing_length, - side_gain) + side_gain, + conv_kernel_params) elif detector == 'NRCBLONG': return NRCBLONGDataset(input_model, odd_even_columns, use_side_ref_pixels, side_smoothing_length, - side_gain) + side_gain, + conv_kernel_params) elif detector == 'NIS': return NIRISSDataset(input_model, odd_even_columns, use_side_ref_pixels, side_smoothing_length, - side_gain) + side_gain, + conv_kernel_params) elif detector == 'GUIDER1': return GUIDER1Dataset(input_model, odd_even_columns, use_side_ref_pixels, side_smoothing_length, - side_gain) + side_gain, + conv_kernel_params) elif detector == 'GUIDER2': return GUIDER2Dataset(input_model, odd_even_columns, use_side_ref_pixels, side_smoothing_length, - side_gain) + side_gain, + conv_kernel_params) else: log.error('Unrecognized detector') return NIRDataset(input_model, odd_even_columns, use_side_ref_pixels, side_smoothing_length, - side_gain) + side_gain, + conv_kernel_params) def correct_model(input_model, odd_even_columns, use_side_ref_pixels, side_smoothing_length, side_gain, - odd_even_rows): + odd_even_rows, + conv_kernel_params): """Wrapper to do Reference Pixel Correction on a JWST Model. Performs the correction on the datamodel @@ -2099,6 +2163,9 @@ def correct_model(input_model, odd_even_columns, flag that controls whether odd and even-numbered rows are handled separately (MIR only) + conv_kernel_params : dict + Dictionary containing the parameters needed for the optimized convolution kernel + """ if input_model.meta.instrument.name == 'MIRI': if reffile_utils.is_subarray(input_model): @@ -2110,7 +2177,8 @@ def correct_model(input_model, odd_even_columns, use_side_ref_pixels, side_smoothing_length, side_gain, - odd_even_rows) + odd_even_rows, + conv_kernel_params) if input_dataset is None: status = SUBARRAY_DOESNTFIT @@ -2138,10 +2206,10 @@ def reference_pixel_correction(input_dataset): Corrected dataset """ - input_dataset.do_corrections() if input_dataset.input_model.meta.exposure.zero_frame: + log.info('Processing the zero frame') process_zeroframe_correction(input_dataset) return diff --git a/jwst/refpix/refpix_step.py b/jwst/refpix/refpix_step.py index e7000ba610..bdcccfde53 100644 --- a/jwst/refpix/refpix_step.py +++ b/jwst/refpix/refpix_step.py @@ -25,12 +25,24 @@ class RefPixStep(Step): ovr_corr_mitigation_ftr = float(default=3.0) # Factor to avoid overcorrection of bad reference pixels for IRS2 preserve_irs2_refpix = boolean(default=False) # Preserve reference pixels in output irs2_mean_subtraction = boolean(default=False) # Apply a mean offset subtraction before IRS2 correction + refpix_algorithm = option("median", "sirs", default="median") # NIR full-frame side pixel algorithm + sigreject = float(default=4.0) # Number of sigmas to reject as outliers + gaussmooth = float(default=1.0) # Width of Gaussian smoothing kernel to use as a low-pass filter + halfwidth = integer(default=30) # Half-width of convolution kernel to build """ reference_file_types = ['refpix'] def process(self, step_input): + conv_kernel_params = { + 'refpix_algorithm': self.refpix_algorithm, + 'sirs_kernel_model': None, + 'sigreject': self.sigreject, + 'gaussmooth': self.gaussmooth, + 'halfwidth': self.halfwidth + } + # Open the input data model with datamodels.RampModel(step_input) as input_model: @@ -51,7 +63,8 @@ def process(self, step_input): self.use_side_ref_pixels = False reference_pixels.correct_model( result, self.odd_even_columns, self.use_side_ref_pixels, - self.side_smoothing_length, self.side_gain, self.odd_even_rows) + self.side_smoothing_length, self.side_gain, self.odd_even_rows, + conv_kernel_params) # Now that values are updated, replace bad reference pixels irs2_subtract_reference.flag_bad_refpix(result, replace_only=True) @@ -81,12 +94,30 @@ def process(self, step_input): else: # Not an NRS IRS2 exposure. Do the normal refpix correction. + + # Get the reference file from CRDS or use user-supplied one only for NIR full-frame data + if self.refpix_algorithm == 'sirs': + if input_model.meta.instrument.name != 'MIRI' and 'FULL' in input_model.meta.subarray.name: + sirs_ref_filename = self.get_reference_file(result, 'sirskernel') + if sirs_ref_filename == 'N/A': + self.log.warning('No reference file found for the optimized convolution kernel.') + self.log.warning('REFPIX step will use the running median algorithm for side pixels.') + else: + self.log.info('Using SIRS reference file: {}'.format(sirs_ref_filename)) + sirs_kernel_model = datamodels.SIRSKernelModel(sirs_ref_filename) + conv_kernel_params['refpix_algorithm'] = sirs_kernel_model + elif input_model.meta.instrument.name == 'MIRI': + self.log.info('Simple Improved Reference Subtraction (SIRS) not applied for MIRI data.') + elif 'FULL' not in input_model.meta.subarray.name: + self.log.info('Simple Improved Reference Subtraction (SIRS) not applied for subarray data.') + status = reference_pixels.correct_model(result, self.odd_even_columns, self.use_side_ref_pixels, self.side_smoothing_length, self.side_gain, - self.odd_even_rows) + self.odd_even_rows, + conv_kernel_params) if status == reference_pixels.REFPIX_OK: result.meta.cal_step.refpix = 'COMPLETE' diff --git a/jwst/refpix/tests/test_optimized_convolution.py b/jwst/refpix/tests/test_optimized_convolution.py new file mode 100644 index 0000000000..9129f62e7e --- /dev/null +++ b/jwst/refpix/tests/test_optimized_convolution.py @@ -0,0 +1,65 @@ +import numpy as np +from stdatamodels.jwst.datamodels import RampModel, SIRSKernelModel +from jwst.refpix.optimized_convolution import make_kernels, get_conv_kernel_coeffs, apply_conv_kernel + + +# create the ConvKernelModel +ckm = {'nrcb1': { + 'gamma': np.array([[0.8737859+0.j, 0.72877103-0.01848215j, 0.7474708+0.00441926j, + 0.7596158-0.01682704j, 0.7710808-0.00618939j], + [0.37835783+0.j, 0.27234325-0.03058944j, 0.38302818+0.03056235j, + 0.36819065-0.02578794j, 0.3908449+0.02115744j], + [0.36443716+0.j, 0.335223+0.02436169j, 0.32699308-0.02325623j, + 0.3830375-0.01340938j, 0.39612782+0.00736016j], + [0.00335188+0.j, 0.01759672-0.01073076j, 0.04302938+0.00353758j, + 0.08149841-0.00643084j, 0.07274915-0.002046j]]), + 'zeta': np.array([[0.14007446+0.0000000e+00j, 0.2371146+1.6455967e-02j, + 0.22727127-5.9413449e-03j, 0.2090475+7.0676603e-03j, + 0.20298977+2.2992526e-05j], + [0.6206608+0.j, 0.680701+0.02468053j, 0.57776874-0.03374288j, + 0.5873975+0.01647749j, 0.5693782-0.02531039j], + [0.6543285+0.j, 0.6167225-0.02665404j, 0.6405862+0.01494319j, + 0.57719606+0.00970044j, 0.57160926-0.01088286j], + [1.0137521+0.j, 0.9492664+0.0071805j, 0.92866725-0.00784425j, + 0.8868761-0.00237024j, 0.89918566-0.00323711j]]) + } + } +conv_kernel_model = SIRSKernelModel(ckm) + + +def mk_data_mdl(data, instrument, detector): + # create input_model + input_model = RampModel(data=data) + input_model.meta.instrument.name = instrument + input_model.meta.instrument.detector = detector + input_model.meta.subarray.name = 'FULL' + return input_model + + +def test_get_conv_kernel_coeffs(): + detector = 'NRCB1' + gamma, zeta = get_conv_kernel_coeffs(conv_kernel_model, detector) + assert gamma is not None + assert zeta is not None + + +def test_mk_kernels(): + detector = 'nothing' + gaussmooth = 1 + halfwidth = 30 + kernels = make_kernels(conv_kernel_model, detector, gaussmooth, halfwidth) + assert kernels is None + + +def test_apply_conv_kernel(): + data = np.zeros((3, 3, 2048, 2048)) + 1.999 + instrument, detector = 'NIRCAM', 'NRCB1' + input_model = mk_data_mdl(data, instrument, detector) + gaussmooth = 1 + halfwidth = 30 + kernels = make_kernels(conv_kernel_model, detector, gaussmooth, halfwidth) + sigreject = 4 + result = apply_conv_kernel(input_model.data[1, 1, ...], kernels, sigreject=sigreject) + compare = np.ones((1, 1, 2048, 2048)) + assert compare.all() == result.all() + diff --git a/jwst/refpix/tests/test_refpix.py b/jwst/refpix/tests/test_refpix.py index f7d5fccaeb..246e8bd488 100644 --- a/jwst/refpix/tests/test_refpix.py +++ b/jwst/refpix/tests/test_refpix.py @@ -8,8 +8,16 @@ Dataset, NIRDataset, correct_model, create_dataset, NRS_edgeless_subarrays) +conv_kernel_params = { + 'refpix_algorithm': 'median', + 'sirs_kernel_model': None, + 'sigreject': 4, + 'gaussmooth': 1, + 'halfwidth': 30} + + def test_refpix_subarray_miri(): - '''Check that the correction is skipped for MIR subarray data ''' + """Check that the correction is skipped for MIR subarray data """ # For MIRI, no reference pixel correction is performed on subarray data # No changes should be seen in the data arrays before and after correction @@ -73,8 +81,8 @@ def test_refpix_subarray_nirspec(subarray, ysize, xsize): def test_each_amp(): - '''Test that each amp is calculated separately using the average of left - and right pixels''' + """Test that each amp is calculated separately using the average of left + and right pixels""" # create input data # create model of data with 0 value array @@ -110,7 +118,7 @@ def test_each_amp(): def test_firstframe_sub(): - '''For MIR data, check that the first group is subtracted from each group in an integration + """For MIR data, check that the first group is subtracted from each group in an integration and added back in after the correction. This was found in testing the amp step. Make sure that the first frame is @@ -118,7 +126,7 @@ def test_firstframe_sub(): in the first group match the reference pixels in all other groups, then the subtraction will result in zeros, leaving zeros to be calculated as the reference pixel values, and the output data will match the input data after the frame is - added back in. So there should be no change to the data.''' + added back in. So there should be no change to the data.""" # create input data # create model of data with 0 value array @@ -150,7 +158,7 @@ def test_firstframe_sub(): np.testing.assert_array_equal(im.data, outim.data) def test_odd_even(): - '''Check that odd/even rows are applied when flag is set''' + """Check that odd/even rows are applied when flag is set""" # Test that odd and even rows are calculated separately @@ -280,7 +288,7 @@ def test_odd_even_amp_nirspec(detector, ysize, odd_even): def test_no_odd_even(): - '''Check that odd/even rows are not applied if flag is set to False''' + """Check that odd/even rows are not applied if flag is set to False""" # Test that odd and even rows are calculated together # create input data @@ -333,8 +341,8 @@ def test_no_odd_even(): def test_side_averaging(): - '''For MIRI data, check that the mean value in the reference pixels is calculated for each amplifier - using the average of the left and right side reference pixels.''' + """For MIRI data, check that the mean value in the reference pixels is calculated for each amplifier + using the average of the left and right side reference pixels.""" # Test that the left and right side pixels are averaged. # create input data @@ -364,8 +372,8 @@ def test_side_averaging(): def test_above_sigma(): - '''Test that a value greater than 3 sigma above mean of reference pixels is rejected - in the averaging of the reference pixels to be subtracted.''' + """Test that a value greater than 3 sigma above mean of reference pixels is rejected + in the averaging of the reference pixels to be subtracted.""" # create input data # create model of data with 0 value array @@ -395,11 +403,11 @@ def test_above_sigma(): def test_nan_refpix(): - '''Verify that the reference pixels flagged DO_NOT_USE are not used in the calculation + """Verify that the reference pixels flagged DO_NOT_USE are not used in the calculation Test that flagging a reference pixel with DO_NOT_USE does not use the pixel in the average. Set the pixel to NaN, which results in a NaN average value if used. If the test - passes, then the NaN was correctly flagged and rejected from the average.''' + passes, then the NaN was correctly flagged and rejected from the average.""" # create input data # create model of data with 0 value array @@ -430,7 +438,7 @@ def test_nan_refpix(): def test_do_corrections_subarray_no_oddEven(setup_subarray_cube): - '''Test all corrections for subarray data with no even/odd.''' + """Test all corrections for subarray data with no even/odd.""" # Create inputs and subarray SUB320A335R data, and set correction parameters ngroups = 3 @@ -462,7 +470,8 @@ def test_do_corrections_subarray_no_oddEven(setup_subarray_cube): use_side_ref_pixels, side_smoothing_length, side_gain, - odd_even_rows) + odd_even_rows, + conv_kernel_params) init_dataset.do_corrections() @@ -472,7 +481,7 @@ def test_do_corrections_subarray_no_oddEven(setup_subarray_cube): def test_do_corrections_subarray(setup_subarray_cube): - '''Test all corrections for subarray data.''' + """Test all corrections for subarray data.""" # Create inputs and subarray SUB320A335R data, and set correction parameters ngroups = 3 @@ -504,7 +513,8 @@ def test_do_corrections_subarray(setup_subarray_cube): use_side_ref_pixels, side_smoothing_length, side_gain, - odd_even_rows) + odd_even_rows, + conv_kernel_params) init_dataset.do_corrections() @@ -514,7 +524,7 @@ def test_do_corrections_subarray(setup_subarray_cube): def test_do_corrections_subarray_4amp(setup_subarray_cube): - '''Test all corrections for subarray data.''' + """Test all corrections for subarray data.""" # Create inputs and subarray SUBGRISM64 data, and set correction parameters ngroups = 3 @@ -575,7 +585,8 @@ def test_do_corrections_subarray_4amp(setup_subarray_cube): use_side_ref_pixels, side_smoothing_length, side_gain, - odd_even_rows) + odd_even_rows, + conv_kernel_params) init_dataset.do_corrections() @@ -583,7 +594,7 @@ def test_do_corrections_subarray_4amp(setup_subarray_cube): def test_get_restore_group_subarray(setup_subarray_cube): - '''Test subarray input model data is replaced with group data.''' + """Test subarray input model data is replaced with group data.""" # Create inputs and subarray SUB320A335R data, and set correction parameters ngroups = 3 @@ -606,6 +617,7 @@ def test_get_restore_group_subarray(setup_subarray_cube): use_side_ref_pixels, side_smoothing_length, side_gain, + conv_kernel_params, odd_even_rows) # Make sure get_group properly copied the subarray @@ -622,7 +634,7 @@ def test_get_restore_group_subarray(setup_subarray_cube): def test_do_top_bottom_correction(setup_cube): - '''Test top/bottom correction for NIRCam data.''' + """Test top/bottom correction for NIRCam data.""" ngroups = 3 nrows = 2048 @@ -639,7 +651,8 @@ def test_do_top_bottom_correction(setup_cube): odd_even_columns, use_side_ref_pixels, side_smoothing_length, - side_gain) + side_gain, + conv_kernel_params) abounds = [0, 512, 1024, 1536, 2048] top_even_amps = [12, 13, 14, 15] @@ -695,7 +708,7 @@ def test_do_top_bottom_correction(setup_cube): def test_do_top_bottom_correction_no_even_odd(setup_cube): - '''Test top/bottom correction with no even/odd.''' + """Test top/bottom correction with no even/odd.""" ngroups = 3 nrows = 2048 @@ -712,7 +725,8 @@ def test_do_top_bottom_correction_no_even_odd(setup_cube): odd_even_columns, use_side_ref_pixels, side_smoothing_length, - side_gain) + side_gain, + conv_kernel_params) abounds = [0, 512, 1024, 1536, 2048] top_amps = [12, 13, 14, 15] @@ -750,7 +764,7 @@ def test_do_top_bottom_correction_no_even_odd(setup_cube): def make_rampmodel(ngroups, ysize, xsize, instrument='MIRI', fill_value=None): - '''Make MIRI or NIRSpec ramp model for testing''' + """Make MIRI or NIRSpec ramp model for testing.""" # create the data and groupdq arrays csize = (1, ngroups, ysize, xsize) @@ -796,7 +810,7 @@ def make_rampmodel(ngroups, ysize, xsize, instrument='MIRI', fill_value=None): @pytest.fixture(scope='function') def setup_cube(): - ''' Set up fake data to test.''' + """ Set up fake data to test.""" def _cube(instr, detector, ngroups, nrows, ncols): @@ -822,7 +836,7 @@ def _cube(instr, detector, ngroups, nrows, ncols): @pytest.fixture(scope='function') def setup_subarray_cube(): - ''' Set up fake NIRCam subarray data to test.''' + """ Set up fake NIRCam subarray data to test.""" def _cube(name, detector, xstart, ystart, ngroups, nrows, ncols): @@ -861,7 +875,7 @@ def _cube(name, detector, xstart, ystart, ngroups, nrows, ncols): ('FGS', "GUIDER2") ]) def test_correct_model(setup_cube, instr, det): - '''Test all corrections for full frame data for all detectors.''' + """Test all corrections for full frame data for all detectors.""" ngroups = 2 nrows = 2048 @@ -885,7 +899,8 @@ def test_correct_model(setup_cube, instr, det): use_side_ref_pixels, side_smoothing_length, side_gain, - odd_even_rows) + odd_even_rows, + conv_kernel_params) np.testing.assert_almost_equal(np.mean(input_model.data[0, 0, :4, 4:-4]), 0, decimal=0) np.testing.assert_almost_equal(np.mean(input_model.data[0, 0, 4:-4, 4:-4]), dataval - rpix, decimal=0) @@ -929,7 +944,8 @@ def test_zero_frame(setup_cube): use_side_ref_pixels, side_smoothing_length, side_gain, - odd_even_rows) + odd_even_rows, + conv_kernel_params) # Make sure the SCI data is as expected. data = np.zeros(input_model.data.shape, dtype=input_model.data.dtype) diff --git a/jwst/regtest/test_nircam_image.py b/jwst/regtest/test_nircam_image.py index 89b08f7de8..40c896367d 100644 --- a/jwst/regtest/test_nircam_image.py +++ b/jwst/regtest/test_nircam_image.py @@ -22,6 +22,7 @@ def run_detector1pipeline(rtdata_module): "--steps.saturation.save_results=True", "--steps.superbias.save_results=True", "--steps.refpix.save_results=True", + "--steps.refpix.refpix_algorithm=median", "--steps.linearity.save_results=True", "--steps.dark_current.save_results=True", "--steps.jump.save_results=True", @@ -30,6 +31,21 @@ def run_detector1pipeline(rtdata_module): Step.from_cmdline(args) +@pytest.fixture(scope="module") +def run_detector1pipeline_with_sirs(rtdata_module): + """Run calwebb_detector1 on NIRCam imaging long data using the convolution kernel algorithm + Simple Improved Reference Subtraction (SIRS)""" + rtdata = rtdata_module + rtdata.get_data("nircam/image/jw01538046001_03105_00001_nrcalong_uncal.fits") + + # Run detector1 pipeline only on one of the _uncal files + args = ["calwebb_detector1", rtdata.input, + "--steps.refpix.refpix_algorithm=sirs", + "--steps.refpix.save_results=True" + ] + Step.from_cmdline(args) + + @pytest.fixture(scope="module") def run_detector1_with_clean_flicker_noise(rtdata_module): """Run detector1 pipeline on NIRCam imaging data with noise cleaning.""" @@ -89,6 +105,22 @@ def run_image3pipeline(run_image2pipeline, rtdata_module): Step.from_cmdline(args) +@pytest.mark.bigdata +def test_nircam_image_sirs(run_detector1pipeline_with_sirs, rtdata_module, fitsdiff_default_kwargs): + """Regression test of detector1 and image2 pipelines performed on NIRCam data.""" + rtdata = rtdata_module + rtdata.input = "jw01538046001_03105_00001_nrcalong_uncal.fits" + output = "jw01538046001_03105_00001_nrcalong_refpix.fits" + rtdata.output = output + rtdata.get_truth("truth/test_nircam_image_stages/jw01538046001_03105_00001_nrcalong_refpix_SIRS.fits") + + fitsdiff_default_kwargs["rtol"] = 5e-5 + fitsdiff_default_kwargs["atol"] = 1e-4 + + diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs) + assert diff.identical, diff.report() + + @pytest.mark.bigdata @pytest.mark.parametrize("suffix", ["dq_init", "saturation", "superbias", "refpix", "linearity", From a4323a05bde4dab2819fbcc32c3a69bfd5afca69 Mon Sep 17 00:00:00 2001 From: Tyler Pauly Date: Fri, 20 Dec 2024 16:42:10 -0500 Subject: [PATCH 4/5] preparation for 1.17.0 release (#9024) --- CHANGES.rst | 270 +++++++++++++++++++++++++++ CITATION.cff | 7 +- README.md | 19 +- changes/8210.general.rst | 1 - changes/8554.assign_wcs.rst | 2 - changes/8554.resample.rst | 1 - changes/8554.skymatch.rst | 1 - changes/8671.docs.rst | 1 - changes/8726.refpix.rst | 1 - changes/8761.stpipe.rst | 1 - changes/8782.outlier_detection.0.rst | 1 - changes/8782.outlier_detection.1.rst | 1 - changes/8782.resample.rst | 1 - changes/8787.background.rst | 1 - changes/8828.outlier_detection.rst | 1 - changes/8831.datamodels.rst | 1 - changes/8840.outlier_detection.rst | 1 - changes/8843.associations.rst | 1 - changes/8846.ami.rst | 1 - changes/8847.master_background.rst | 1 - changes/8849.emicorr.rst | 1 - changes/8851.outlier_detection.rst | 1 - changes/8852.general.rst | 1 - changes/8853.outlier_detection.rst | 1 - changes/8866.resample.rst | 1 - changes/8870.outlier_detection.rst | 1 - changes/8874.assign_wcs.rst | 1 - changes/8880.outlier_detection.rst | 1 - changes/8885.general.rst | 1 - changes/8890.jump.rst | 2 - changes/8892.skymatch.rst | 1 - changes/8893.resample.rst | 1 - changes/8897.assign_wcs.rst | 1 - changes/8907.pipeline.rst | 1 - changes/8908.resample_spec.rst | 1 - changes/8909.scripts.rst | 1 - changes/8911.cube_build.rst | 1 - changes/8913.cube_build.rst | 1 - changes/8916.background.rst | 1 - changes/8918.datamodels.rst | 1 - changes/8926.docs.rst | 1 - changes/8927.combine_1d.rst | 1 - changes/8927.master_background.rst | 1 - changes/8927.pipeline.rst | 1 - changes/8932.master_background.rst | 1 - changes/8935.general.rst | 1 - changes/8945.stpipe.rst | 1 - changes/8946.rscd.rst | 1 - changes/8947.pipeline.rst | 1 - changes/8950.general.rst | 1 - changes/8952.firstframe.rst | 1 - changes/8957.general.rst | 1 - changes/8958.general.rst | 1 - changes/8961.extract_1d.rst | 1 - changes/8963.assign_wcs.rst | 3 - changes/8966.general.rst | 1 - changes/8974.ami.rst | 1 - changes/8975.resample.rst | 1 - changes/8978.assign_wcs.rst | 1 - changes/8983.extract_1d.rst | 1 - changes/8990.background.rst | 1 - pyproject.toml | 6 +- requirements-sdp.txt | 78 ++++++++ 63 files changed, 366 insertions(+), 76 deletions(-) delete mode 100644 changes/8210.general.rst delete mode 100644 changes/8554.assign_wcs.rst delete mode 100644 changes/8554.resample.rst delete mode 100644 changes/8554.skymatch.rst delete mode 100644 changes/8671.docs.rst delete mode 100644 changes/8726.refpix.rst delete mode 100644 changes/8761.stpipe.rst delete mode 100644 changes/8782.outlier_detection.0.rst delete mode 100644 changes/8782.outlier_detection.1.rst delete mode 100644 changes/8782.resample.rst delete mode 100644 changes/8787.background.rst delete mode 100644 changes/8828.outlier_detection.rst delete mode 100644 changes/8831.datamodels.rst delete mode 100644 changes/8840.outlier_detection.rst delete mode 100644 changes/8843.associations.rst delete mode 100644 changes/8846.ami.rst delete mode 100644 changes/8847.master_background.rst delete mode 100644 changes/8849.emicorr.rst delete mode 100644 changes/8851.outlier_detection.rst delete mode 100644 changes/8852.general.rst delete mode 100644 changes/8853.outlier_detection.rst delete mode 100644 changes/8866.resample.rst delete mode 100644 changes/8870.outlier_detection.rst delete mode 100644 changes/8874.assign_wcs.rst delete mode 100644 changes/8880.outlier_detection.rst delete mode 100644 changes/8885.general.rst delete mode 100644 changes/8890.jump.rst delete mode 100644 changes/8892.skymatch.rst delete mode 100644 changes/8893.resample.rst delete mode 100644 changes/8897.assign_wcs.rst delete mode 100644 changes/8907.pipeline.rst delete mode 100644 changes/8908.resample_spec.rst delete mode 100644 changes/8909.scripts.rst delete mode 100644 changes/8911.cube_build.rst delete mode 100644 changes/8913.cube_build.rst delete mode 100644 changes/8916.background.rst delete mode 100644 changes/8918.datamodels.rst delete mode 100644 changes/8926.docs.rst delete mode 100644 changes/8927.combine_1d.rst delete mode 100644 changes/8927.master_background.rst delete mode 100644 changes/8927.pipeline.rst delete mode 100644 changes/8932.master_background.rst delete mode 100644 changes/8935.general.rst delete mode 100644 changes/8945.stpipe.rst delete mode 100644 changes/8946.rscd.rst delete mode 100644 changes/8947.pipeline.rst delete mode 100644 changes/8950.general.rst delete mode 100644 changes/8952.firstframe.rst delete mode 100644 changes/8957.general.rst delete mode 100644 changes/8958.general.rst delete mode 100644 changes/8961.extract_1d.rst delete mode 100644 changes/8963.assign_wcs.rst delete mode 100644 changes/8966.general.rst delete mode 100644 changes/8974.ami.rst delete mode 100644 changes/8975.resample.rst delete mode 100644 changes/8978.assign_wcs.rst delete mode 100644 changes/8983.extract_1d.rst delete mode 100644 changes/8990.background.rst diff --git a/CHANGES.rst b/CHANGES.rst index 1041652782..55f5f2c371 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,273 @@ +1.17.0 (2024-12-20) +=================== + +General +------- + +- Rename default branch to main. (`#8210 + `_) +- Added mypy type checking to CI checks (`#8852 + `_) +- Give regtest okify results unique subdirectories. (`#8885 + `_) +- Include xml and db test data as package data for lib tests. (`#8935 + `_) +- Update minimum required version of crds to allow for "data release style" + contexts. (`#8950 `_) +- remove ``okify_regtests`` script (move to ``ci_watson``) (`#8957 + `_) +- When blending metadata don't store columns containing all missing value + (nans). (`#8958 `_) +- Increase asdf upper pin to 5 (`#8966 + `_) + + +Documentation +------------- + +- use ``towncrier`` to handle change log entries (`#8671 + `_) +- Mention possible need to provide package name to strun when using aliases. + (`#8926 `_) + + +``stpipe`` +---------- + +- Add warning that ``Step.__call__`` is deprecated. (`#8761 + `_) +- Remove all uses of Step.__call__ to allow it's deprecation. (`#8945 + `_) + + +Data Models +----------- + +- Remove memory-saving options from ModelContainer (`#8831 + `_) +- Update ModelLibrary to use meta.asn.exptype instead of meta.exptype. (`#8918 + `_) + + +Associations +------------ + +- Add mosaic association candidates to list of level three candidate types + requiring members from more than one observation (`#8843 + `_) + + +Scripts +------- + +- Remove the outdated schema_editor script. (`#8909 + `_) + + +Pipeline +-------- + +- Fixed a bug leading to incorrect area extensions, and sometimes crashes, in + the coron3 pipeline (`#8907 + `_) +- Add 'mbsub' to the list of known suffixes, for ``master_background`` + correction in ``calwebb_spec3``. (`#8927 + `_) +- Transfer wcsinfo metadata to new MultiSlitModel created during + ``calwebb_spec2`` processing of NIRSpec MSA data. (`#8947 + `_) + + +ami_analyze / ami_normalize / ami_average (ami3) +------------------------------------------------ + +- Change how AMI observables are averaged: average fringe quantities before + calculating additional observables. Update their error calculation: use + covariance of amplitudes/phases (and derived quantities) and standard error + of the mean. Code now expects an ASDF filename string for user-supplied + affine2d and bandpass input arguments. Example file creation in + documentation. (`#8846 + `_) +- Use mask and pupil geometry constants from NRM reference file, and apply + affine distortion from commissioning to LG model as default. (`#8974 + `_) + + +assign_wcs (image2, spec2) +-------------------------- + +- Use the range of points in the TabularModel to adjust the bounding_box in a + MIRI LRS FIXEDSLIT observation. + Ignore the bounding_box in running the inverse WCS transform when computing + crpix. (`#8554 `_) +- Catch NaN values in msa tables for source positions in slit and replace with + slit center. (`#8874 `_) +- Use pixel vertices to define s_region instead of pixel centers." (`#8897 + `_) +- Fix all bounding box assignments to wcs objects so that they are correctly + and + specifically assigned as order ``F`` boxes. This ensures consistency with the + assumptions made by GWCS for bounding boxes. (`#8963 + `_) +- Fix negative SHUTTRID values under numpy 2 (`#8978 + `_) + + +background (image2, spec2) +-------------------------- + +- Fixed crash when combining full and subarray observations for background + subtraction. (`#8787 `_) +- Apply bitwise operations in correct order when counting good pixels in the + background mask during WFSS background subtraction. (`#8916 + `_) +- Compute scaling for WFSS background subtraction using error-weighted mean + (`#8990 `_) + + +combine_1d (spec3) +------------------ + +- Fix wavelength sort order for single input spectrum. (`#8927 + `_) + + +cube_build (spec2 IFU, spec3) +----------------------------- + +- For moving-target IFU data, set RA, Dec header information of s3d products + according to the mean of input models instead of the first input model. + (`#8911 `_) +- Tweak the cube_build spaxel debugging option to provide filename info and + match outputs to the stated ordering. (`#8913 + `_) + + +emicorr (detector1 MIR) +----------------------- + +- Removed the interleaved noise array and do interleaving of noise and + subtraction in-place to avoid creating 2 arrays of equal dimensions to data. + (`#8849 `_) + + +extract_1d (spec2, spec3) +------------------------- + +- Refactor the core extraction algorithm and aperture definition modules for + slit and slitless extractions, for greater efficiency and maintainability. + Extraction reference files in FITS format are no longer supported. Current + behavior for extractions proceeding from extract1d reference files in JSON + format is preserved, with minor improvements: DQ arrays are populated and + error propagation is improved for some aperture types. (`#8961 + `_) +- Set order 2 weights to 1.0 for NIRISS SOSS SUBSTRIP96 data to avoid a + spurious flux drop in spectra around 1.5um. (`#8983 + `_) + + +firstframe (detector1 MIR) +-------------------------- + +- Update the firstframe step to optionally not flag when a ramp saturates in + group 3 (`#8952 `_) + + +jump (detector1) +---------------- + +- Add maximum_shower_amplitude parameter to MIRI cosmic rays showers routine + to fix accidental flagging of bright science pixels. (`#8890 + `_) + + +master_background (spec2 MOS, spec3) +------------------------------------ + +- Include resample, pixel_replace and extract_1d into MOS background pipeline + (`#8847 `_) +- Fix ModelContainer handling for user background input. (`#8927 + `_) +- Added pixel-to-pixel sigma clipping on input backgrounds in the NIRSpec MOS + master_background_mos step. Added option to median filter master background + spectrum in both the master_background and master_background_mos steps. + (`#8932 `_) + + +outlier_detection (image3, tso3, spec3, coron3) +----------------------------------------------- + +- Fix a bug that caused intermediate files to conflict for different slits when + a MultiSlitModel was processed. (`#8782 + `_) +- Decrease the amount of file I/O required to compute the median when in_memory + is set to False. (`#8782 + `_) +- For slit spectra, threshold outliers with a median error across exposures + instead of input error from the exposure itself. (`#8828 + `_) +- Moved median computers out of the jwst repository and into stcal. (`#8840 + `_) +- Update documentation to clarify the interaction between pipeline-level and + step-level `--in_memory` flags. (`#8851 + `_) +- Avoid modifying input and saving duplicate files when resample_data=False. + (`#8853 `_) +- Remove deprecated nlow and nhigh parameters from outlier detection step. + (`#8870 `_) +- reorganize outlier detection documentation (`#8880 + `_) + + +refpix (detector1) +------------------ + +- Implemented SIRS algorithm instead of running median for side pixels of NIR + full-frame data. Running median is still default. (`#8726 + `_) + + +resample (image2, image3, coron3) +--------------------------------- + +- Ignore the bounding_box in the inverse WCS transform in reproject. (`#8554 + `_) +- Permit creating drizzled models one at a time in many-to-many mode. (`#8782 + `_) +- Updated resample code to support the new ``drizzle`` API, see + https://github.com/spacetelescope/drizzle/pull/134 for more details. (`#8866 + `_) +- Use s_region list to calculate output footprint instead of re-computing via + WCS transforms (`#8893 + `_) +- Removed allowed_memory parameter and DMODEL_ALLOWED_MEMORY environment + variable (`#8975 `_) + + +resample_spec (spec2 MOS, spec3) +-------------------------------- + +- Update NIRSpec spectral resampling to add a missing correction factor in + resampled WCS tangent plane transformation. (`#8908 + `_) + + +rscd (detector1 MIR) +-------------------- + +- Updated RSCD step to work on segmented data (`#8946 + `_) + + +skymatch (image3) +----------------- + +- Ignore the bounding_box of an observation when computing sky statistics. + (`#8554 `_) +- Resolve warnings emitted by NumPy 2 when running skymatch. (`#8892 + `_) + + 1.16.1 (2024-10-30) =================== diff --git a/CITATION.cff b/CITATION.cff index 6d933fee0e..c660ccdba4 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -92,8 +92,11 @@ authors: - family-names: "Clarke" given-names: "Melanie" orcid: "https://orcid.org/0009-0002-3561-4347" +- family-names: "Filippazzo" + given-names: "Joseph" + orcid: "https://orcid.org/0000-0002-0201-8306" title: "JWST Calibration Pipeline" -version: 1.16.0 +version: 1.17.0 doi: 10.5281/zenodo.7038885 -date-released: 2024-09-20 +date-released: 2024-12-20 url: "https://github.com/spacetelescope/jwst" diff --git a/README.md b/README.md index 84d50d76c8..91be40ef59 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ You can also install a specific version: conda create -n python=3.11 conda activate - pip install jwst==1.9.4 + pip install jwst==1.16.1 ### Installing the development version from Github @@ -89,15 +89,15 @@ used for Linux and Mac OS systems. Linux: - conda create -n jwstdp-1.12.5 --file https://ssb.stsci.edu/releases/jwstdp/1.12.5/conda_python_stable-deps.txt - conda activate jwstdp-1.12.5 - pip install -r https://ssb.stsci.edu/releases/jwstdp/1.12.5/reqs_stable-deps.txt + conda create -n jwstdp-1.16.1 --file https://ssb.stsci.edu/releases/jwstdp/1.16.1/conda_python_stable-deps.txt + conda activate jwstdp-1.16.1 + pip install -r https://ssb.stsci.edu/releases/jwstdp/1.16.1/reqs_stable-deps.txt MacOS: - conda create -n jwstdp-1.12.5 --file https://ssb.stsci.edu/releases/jwstdp/1.12.5/conda_python_macos-stable-deps.txt - conda activate jwstdp-1.12.5 - pip install -r https://ssb.stsci.edu/releases/jwstdp/1.12.5/reqs_macos-stable-deps.txt + conda create -n jwstdp-1.16.1 --file https://ssb.stsci.edu/releases/jwstdp/1.16.1/conda_python_macos-stable-deps.txt + conda activate jwstdp-1.16.1 + pip install -r https://ssb.stsci.edu/releases/jwstdp/1.16.1/reqs_macos-stable-deps.txt Each DMS delivery has its own installation instructions, which may be found in the corresponding release documentation linked from this page: @@ -216,8 +216,9 @@ the specified context and less than the context for the next release. | jwst tag | DMS build | SDP_VER | CRDS_CONTEXT | Released | Ops Install | Notes | |---------------------|-----------|----------|--------------|------------|-------------|-----------------------------------------------| -| 1.16.1 | B11.1.1 | 2024.3.1 | 1298 | 2024-11-13 | TBD | Final release candidate for B11.1 | -| 1.16.0 | B11.1 | 2024.3.0 | 1298 | 2024-09-20 | TBD | First release candidate for B11.1 | +| 1.17.0rc1 | B11.2 | TBD | 1321 | 2024-12-20 | TBD | First release candidate for B11.2 | +| 1.16.1 | B11.1.1 | 2024.3.1 | 1303 | 2024-11-13 | 2024-12-06 | Final release candidate for B11.1 | +| 1.16.0 | B11.1 | 2024.3.0 | 1298 | 2024-09-20 | | First release candidate for B11.1 | | 1.15.1 | B11.0 | 2024.2.2 | 1293 | 2024-07-08 | 2024-09-12 | Final release candidate for B11.0 | | 1.15.0 | B11.0rc1 | | 1274 | 2024-06-26 | | First release candidate for B11.0 | | 1.14.1 | | | 1240 | 2024-06-27 | | PyPI-only release for external users | diff --git a/changes/8210.general.rst b/changes/8210.general.rst deleted file mode 100644 index b923267b54..0000000000 --- a/changes/8210.general.rst +++ /dev/null @@ -1 +0,0 @@ -Rename default branch to main. diff --git a/changes/8554.assign_wcs.rst b/changes/8554.assign_wcs.rst deleted file mode 100644 index c36dd9d167..0000000000 --- a/changes/8554.assign_wcs.rst +++ /dev/null @@ -1,2 +0,0 @@ -Use the range of points in the TabularModel to adjust the bounding_box in a MIRI LRS FIXEDSLIT observation. -Ignore the bounding_box in running the inverse WCS transform when computing crpix. diff --git a/changes/8554.resample.rst b/changes/8554.resample.rst deleted file mode 100644 index bda0397ae8..0000000000 --- a/changes/8554.resample.rst +++ /dev/null @@ -1 +0,0 @@ -Ignore the bounding_box in the inverse WCS transform in reproject. diff --git a/changes/8554.skymatch.rst b/changes/8554.skymatch.rst deleted file mode 100644 index 1561af36b6..0000000000 --- a/changes/8554.skymatch.rst +++ /dev/null @@ -1 +0,0 @@ -Ignore the bounding_box of an observation when computing sky statistics. diff --git a/changes/8671.docs.rst b/changes/8671.docs.rst deleted file mode 100644 index 724817599d..0000000000 --- a/changes/8671.docs.rst +++ /dev/null @@ -1 +0,0 @@ -use ``towncrier`` to handle change log entries diff --git a/changes/8726.refpix.rst b/changes/8726.refpix.rst deleted file mode 100644 index 0639bf5208..0000000000 --- a/changes/8726.refpix.rst +++ /dev/null @@ -1 +0,0 @@ -Implemented SIRS algorithm instead of running median for side pixels of NIR full-frame data. Running median is still default. diff --git a/changes/8761.stpipe.rst b/changes/8761.stpipe.rst deleted file mode 100644 index 3d2985c04f..0000000000 --- a/changes/8761.stpipe.rst +++ /dev/null @@ -1 +0,0 @@ -Add warning that ``Step.__call__`` is deprecated. diff --git a/changes/8782.outlier_detection.0.rst b/changes/8782.outlier_detection.0.rst deleted file mode 100644 index 469e927e44..0000000000 --- a/changes/8782.outlier_detection.0.rst +++ /dev/null @@ -1 +0,0 @@ -Decrease the amount of file I/O required to compute the median when in_memory is set to False. \ No newline at end of file diff --git a/changes/8782.outlier_detection.1.rst b/changes/8782.outlier_detection.1.rst deleted file mode 100644 index a1f9bd8e55..0000000000 --- a/changes/8782.outlier_detection.1.rst +++ /dev/null @@ -1 +0,0 @@ -Fix a bug that caused intermediate files to conflict for different slits when a MultiSlitModel was processed. \ No newline at end of file diff --git a/changes/8782.resample.rst b/changes/8782.resample.rst deleted file mode 100644 index 05b3e87052..0000000000 --- a/changes/8782.resample.rst +++ /dev/null @@ -1 +0,0 @@ -Permit creating drizzled models one at a time in many-to-many mode. diff --git a/changes/8787.background.rst b/changes/8787.background.rst deleted file mode 100644 index 9f04f7e0bd..0000000000 --- a/changes/8787.background.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed crash when combining full and subarray observations for background subtraction. diff --git a/changes/8828.outlier_detection.rst b/changes/8828.outlier_detection.rst deleted file mode 100644 index e0edcecb80..0000000000 --- a/changes/8828.outlier_detection.rst +++ /dev/null @@ -1 +0,0 @@ -For slit spectra, threshold outliers with a median error across exposures instead of input error from the exposure itself. diff --git a/changes/8831.datamodels.rst b/changes/8831.datamodels.rst deleted file mode 100644 index 22289006df..0000000000 --- a/changes/8831.datamodels.rst +++ /dev/null @@ -1 +0,0 @@ -Remove memory-saving options from ModelContainer \ No newline at end of file diff --git a/changes/8840.outlier_detection.rst b/changes/8840.outlier_detection.rst deleted file mode 100644 index 0d1fe91bb7..0000000000 --- a/changes/8840.outlier_detection.rst +++ /dev/null @@ -1 +0,0 @@ -Moved median computers out of the jwst repository and into stcal. \ No newline at end of file diff --git a/changes/8843.associations.rst b/changes/8843.associations.rst deleted file mode 100644 index e6d2334960..0000000000 --- a/changes/8843.associations.rst +++ /dev/null @@ -1 +0,0 @@ -Add mosaic association candidates to list of level three candidate types requiring members from more than one observation \ No newline at end of file diff --git a/changes/8846.ami.rst b/changes/8846.ami.rst deleted file mode 100644 index a965c823ff..0000000000 --- a/changes/8846.ami.rst +++ /dev/null @@ -1 +0,0 @@ -Change how AMI observables are averaged: average fringe quantities before calculating additional observables. Update their error calculation: use covariance of amplitudes/phases (and derived quantities) and standard error of the mean. Code now expects an ASDF filename string for user-supplied affine2d and bandpass input arguments. Example file creation in documentation. \ No newline at end of file diff --git a/changes/8847.master_background.rst b/changes/8847.master_background.rst deleted file mode 100644 index f1b516a1f2..0000000000 --- a/changes/8847.master_background.rst +++ /dev/null @@ -1 +0,0 @@ -Include resample, pixel_replace and extract_1d into MOS background pipeline diff --git a/changes/8849.emicorr.rst b/changes/8849.emicorr.rst deleted file mode 100644 index aee1210d87..0000000000 --- a/changes/8849.emicorr.rst +++ /dev/null @@ -1 +0,0 @@ -Removed the interleaved noise array and do interleaving of noise and subtraction in-place to avoid creating 2 arrays of equal dimensions to data. diff --git a/changes/8851.outlier_detection.rst b/changes/8851.outlier_detection.rst deleted file mode 100644 index da2e49986d..0000000000 --- a/changes/8851.outlier_detection.rst +++ /dev/null @@ -1 +0,0 @@ -Update documentation to clarify the interaction between pipeline-level and step-level `--in_memory` flags. \ No newline at end of file diff --git a/changes/8852.general.rst b/changes/8852.general.rst deleted file mode 100644 index 0a584e8d63..0000000000 --- a/changes/8852.general.rst +++ /dev/null @@ -1 +0,0 @@ -Added mypy type checking to CI checks diff --git a/changes/8853.outlier_detection.rst b/changes/8853.outlier_detection.rst deleted file mode 100644 index abb775f6c8..0000000000 --- a/changes/8853.outlier_detection.rst +++ /dev/null @@ -1 +0,0 @@ -Avoid modifying input and saving duplicate files when resample_data=False. diff --git a/changes/8866.resample.rst b/changes/8866.resample.rst deleted file mode 100644 index f31024d0ee..0000000000 --- a/changes/8866.resample.rst +++ /dev/null @@ -1 +0,0 @@ -Updated resample code to support the new ``drizzle`` API, see https://github.com/spacetelescope/drizzle/pull/134 for more details. diff --git a/changes/8870.outlier_detection.rst b/changes/8870.outlier_detection.rst deleted file mode 100644 index 53ce01221d..0000000000 --- a/changes/8870.outlier_detection.rst +++ /dev/null @@ -1 +0,0 @@ -Remove deprecated nlow and nhigh parameters from outlier detection step. diff --git a/changes/8874.assign_wcs.rst b/changes/8874.assign_wcs.rst deleted file mode 100644 index d27218f8d4..0000000000 --- a/changes/8874.assign_wcs.rst +++ /dev/null @@ -1 +0,0 @@ -Catch NaN values in msa tables for source positions in slit and replace with slit center. \ No newline at end of file diff --git a/changes/8880.outlier_detection.rst b/changes/8880.outlier_detection.rst deleted file mode 100644 index f5642b4e30..0000000000 --- a/changes/8880.outlier_detection.rst +++ /dev/null @@ -1 +0,0 @@ -reorganize outlier detection documentation diff --git a/changes/8885.general.rst b/changes/8885.general.rst deleted file mode 100644 index 435de02426..0000000000 --- a/changes/8885.general.rst +++ /dev/null @@ -1 +0,0 @@ -Give regtest okify results unique subdirectories. diff --git a/changes/8890.jump.rst b/changes/8890.jump.rst deleted file mode 100644 index d04413a547..0000000000 --- a/changes/8890.jump.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add maximum_shower_amplitude parameter to MIRI cosmic rays showers routine -to fix accidental flagging of bright science pixels. diff --git a/changes/8892.skymatch.rst b/changes/8892.skymatch.rst deleted file mode 100644 index 7662782814..0000000000 --- a/changes/8892.skymatch.rst +++ /dev/null @@ -1 +0,0 @@ -Resolve warnings emitted by NumPy 2 when running skymatch. diff --git a/changes/8893.resample.rst b/changes/8893.resample.rst deleted file mode 100644 index 9e48ecea89..0000000000 --- a/changes/8893.resample.rst +++ /dev/null @@ -1 +0,0 @@ -Use s_region list to calculate output footprint instead of re-computing via WCS transforms diff --git a/changes/8897.assign_wcs.rst b/changes/8897.assign_wcs.rst deleted file mode 100644 index fb209a0fdc..0000000000 --- a/changes/8897.assign_wcs.rst +++ /dev/null @@ -1 +0,0 @@ -Use pixel vertices to define s_region instead of pixel centers." diff --git a/changes/8907.pipeline.rst b/changes/8907.pipeline.rst deleted file mode 100644 index c6d1759f93..0000000000 --- a/changes/8907.pipeline.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed a bug leading to incorrect area extensions, and sometimes crashes, in the coron3 pipeline diff --git a/changes/8908.resample_spec.rst b/changes/8908.resample_spec.rst deleted file mode 100644 index 2d69d9feee..0000000000 --- a/changes/8908.resample_spec.rst +++ /dev/null @@ -1 +0,0 @@ -Update NIRSpec spectral resampling to add a missing correction factor in resampled WCS tangent plane transformation. \ No newline at end of file diff --git a/changes/8909.scripts.rst b/changes/8909.scripts.rst deleted file mode 100644 index 0addc426d1..0000000000 --- a/changes/8909.scripts.rst +++ /dev/null @@ -1 +0,0 @@ -Remove the outdated schema_editor script. diff --git a/changes/8911.cube_build.rst b/changes/8911.cube_build.rst deleted file mode 100644 index e6de4d2827..0000000000 --- a/changes/8911.cube_build.rst +++ /dev/null @@ -1 +0,0 @@ -For moving-target IFU data, set RA, Dec header information of s3d products according to the mean of input models instead of the first input model. diff --git a/changes/8913.cube_build.rst b/changes/8913.cube_build.rst deleted file mode 100644 index 2fd2ff006f..0000000000 --- a/changes/8913.cube_build.rst +++ /dev/null @@ -1 +0,0 @@ -Tweak the cube_build spaxel debugging option to provide filename info and match outputs to the stated ordering. diff --git a/changes/8916.background.rst b/changes/8916.background.rst deleted file mode 100644 index e2e4414b14..0000000000 --- a/changes/8916.background.rst +++ /dev/null @@ -1 +0,0 @@ -Apply bitwise operations in correct order when counting good pixels in the background mask during WFSS background subtraction. \ No newline at end of file diff --git a/changes/8918.datamodels.rst b/changes/8918.datamodels.rst deleted file mode 100644 index 2d442399f7..0000000000 --- a/changes/8918.datamodels.rst +++ /dev/null @@ -1 +0,0 @@ -Update ModelLibrary to use meta.asn.exptype instead of meta.exptype. diff --git a/changes/8926.docs.rst b/changes/8926.docs.rst deleted file mode 100644 index 621c1a9254..0000000000 --- a/changes/8926.docs.rst +++ /dev/null @@ -1 +0,0 @@ -Mention possible need to provide package name to strun when using aliases. diff --git a/changes/8927.combine_1d.rst b/changes/8927.combine_1d.rst deleted file mode 100644 index b05b3e95bc..0000000000 --- a/changes/8927.combine_1d.rst +++ /dev/null @@ -1 +0,0 @@ -Fix wavelength sort order for single input spectrum. diff --git a/changes/8927.master_background.rst b/changes/8927.master_background.rst deleted file mode 100644 index cf71da1e52..0000000000 --- a/changes/8927.master_background.rst +++ /dev/null @@ -1 +0,0 @@ -Fix ModelContainer handling for user background input. diff --git a/changes/8927.pipeline.rst b/changes/8927.pipeline.rst deleted file mode 100644 index e03ac78254..0000000000 --- a/changes/8927.pipeline.rst +++ /dev/null @@ -1 +0,0 @@ -Add 'mbsub' to the list of known suffixes, for ``master_background`` correction in ``calwebb_spec3``. diff --git a/changes/8932.master_background.rst b/changes/8932.master_background.rst deleted file mode 100644 index 4ad13be5d5..0000000000 --- a/changes/8932.master_background.rst +++ /dev/null @@ -1 +0,0 @@ -Added pixel-to-pixel sigma clipping on input backgrounds in the NIRSpec MOS master_background_mos step. Added option to median filter master background spectrum in both the master_background and master_background_mos steps. diff --git a/changes/8935.general.rst b/changes/8935.general.rst deleted file mode 100644 index fe0b6a58e2..0000000000 --- a/changes/8935.general.rst +++ /dev/null @@ -1 +0,0 @@ -Include xml and db test data as package data for lib tests. diff --git a/changes/8945.stpipe.rst b/changes/8945.stpipe.rst deleted file mode 100644 index 73937c9f0f..0000000000 --- a/changes/8945.stpipe.rst +++ /dev/null @@ -1 +0,0 @@ -Remove all uses of Step.__call__ to allow it's deprecation. diff --git a/changes/8946.rscd.rst b/changes/8946.rscd.rst deleted file mode 100644 index 5ef30ed606..0000000000 --- a/changes/8946.rscd.rst +++ /dev/null @@ -1 +0,0 @@ -Updated RSCD step to work on segmented data diff --git a/changes/8947.pipeline.rst b/changes/8947.pipeline.rst deleted file mode 100644 index 2c24cc6f0b..0000000000 --- a/changes/8947.pipeline.rst +++ /dev/null @@ -1 +0,0 @@ -Transfer wcsinfo metadata to new MultiSlitModel created during ``calwebb_spec2`` processing of NIRSpec MSA data. \ No newline at end of file diff --git a/changes/8950.general.rst b/changes/8950.general.rst deleted file mode 100644 index 466db73740..0000000000 --- a/changes/8950.general.rst +++ /dev/null @@ -1 +0,0 @@ -Update minimum required version of crds to allow for "data release style" contexts. diff --git a/changes/8952.firstframe.rst b/changes/8952.firstframe.rst deleted file mode 100644 index ff41581d2f..0000000000 --- a/changes/8952.firstframe.rst +++ /dev/null @@ -1 +0,0 @@ -Update the firstframe step to optionally not flag when a ramp saturates in group 3 \ No newline at end of file diff --git a/changes/8957.general.rst b/changes/8957.general.rst deleted file mode 100644 index 91a475dc0f..0000000000 --- a/changes/8957.general.rst +++ /dev/null @@ -1 +0,0 @@ -remove ``okify_regtests`` script (move to ``ci_watson``) diff --git a/changes/8958.general.rst b/changes/8958.general.rst deleted file mode 100644 index a84befe1ff..0000000000 --- a/changes/8958.general.rst +++ /dev/null @@ -1 +0,0 @@ -When blending metadata don't store columns containing all missing value (nans). diff --git a/changes/8961.extract_1d.rst b/changes/8961.extract_1d.rst deleted file mode 100644 index ce89d4c2b9..0000000000 --- a/changes/8961.extract_1d.rst +++ /dev/null @@ -1 +0,0 @@ -Refactor the core extraction algorithm and aperture definition modules for slit and slitless extractions, for greater efficiency and maintainability. Extraction reference files in FITS format are no longer supported. Current behavior for extractions proceeding from extract1d reference files in JSON format is preserved, with minor improvements: DQ arrays are populated and error propagation is improved for some aperture types. diff --git a/changes/8963.assign_wcs.rst b/changes/8963.assign_wcs.rst deleted file mode 100644 index b9f007448a..0000000000 --- a/changes/8963.assign_wcs.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix all bounding box assignments to wcs objects so that they are correctly and -specifically assigned as order ``F`` boxes. This ensures consistency with the -assumptions made by GWCS for bounding boxes. diff --git a/changes/8966.general.rst b/changes/8966.general.rst deleted file mode 100644 index 5b21ca6897..0000000000 --- a/changes/8966.general.rst +++ /dev/null @@ -1 +0,0 @@ -Increase asdf upper pin to 5 diff --git a/changes/8974.ami.rst b/changes/8974.ami.rst deleted file mode 100644 index ea39be8806..0000000000 --- a/changes/8974.ami.rst +++ /dev/null @@ -1 +0,0 @@ -Use mask and pupil geometry constants from NRM reference file, and apply affine distortion from commissioning to LG model as default. \ No newline at end of file diff --git a/changes/8975.resample.rst b/changes/8975.resample.rst deleted file mode 100644 index 95d48e855e..0000000000 --- a/changes/8975.resample.rst +++ /dev/null @@ -1 +0,0 @@ -Removed allowed_memory parameter and DMODEL_ALLOWED_MEMORY environment variable diff --git a/changes/8978.assign_wcs.rst b/changes/8978.assign_wcs.rst deleted file mode 100644 index 2771c1d3c6..0000000000 --- a/changes/8978.assign_wcs.rst +++ /dev/null @@ -1 +0,0 @@ -Fix negative SHUTTRID values under numpy 2 diff --git a/changes/8983.extract_1d.rst b/changes/8983.extract_1d.rst deleted file mode 100644 index 5bf93662f2..0000000000 --- a/changes/8983.extract_1d.rst +++ /dev/null @@ -1 +0,0 @@ -Set order 2 weights to 1.0 for NIRISS SOSS SUBSTRIP96 data to avoid a spurious flux drop in spectra around 1.5um. diff --git a/changes/8990.background.rst b/changes/8990.background.rst deleted file mode 100644 index ed656009a4..0000000000 --- a/changes/8990.background.rst +++ /dev/null @@ -1 +0,0 @@ -Compute scaling for WFSS background subtraction using error-weighted mean diff --git a/pyproject.toml b/pyproject.toml index 12c5996d16..3342f6b90e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,9 +33,9 @@ dependencies = [ "scikit-image>=0.20.0", "scipy>=1.14.1", "spherical-geometry>=1.2.22", - "stcal @ git+https://github.com/spacetelescope/stcal.git@main", - "stdatamodels @ git+https://github.com/spacetelescope/stdatamodels.git@main", - "stpipe @ git+https://github.com/spacetelescope/stpipe.git@main", + "stcal>=1.11.0,<1.12.0", + "stdatamodels>=2.2.0,<2.3.0", + "stpipe>=0.8.0,<0.9.0", "stsci.imagestats>=1.6.3", "synphot>=1.2", "tweakwcs>=0.8.8", diff --git a/requirements-sdp.txt b/requirements-sdp.txt index a110a4b011..f6f1a9c478 100644 --- a/requirements-sdp.txt +++ b/requirements-sdp.txt @@ -14,3 +14,81 @@ # conda activate sdp # pip install -e .[test,sdp] # pip freeze | grep -v jwst >> requirements-sdp.txt +asdf==4.0.0 +asdf-astropy==0.7.0 +asdf_coordinates_schemas==0.3.0 +asdf_standard==1.1.1 +asdf_transform_schemas==0.5.0 +asdf_wcs_schemas==0.4.0 +astropy==7.0.0 +astropy-iers-data==0.2024.12.16.0.35.48 +attrs==24.3.0 +BayesicFitting==3.2.3 +certifi==2024.12.14 +charset-normalizer==3.4.0 +ci_watson==0.8.0 +colorama==0.4.6 +contourpy==1.3.1 +coverage==7.6.9 +crds==12.0.8 +cycler==0.12.1 +drizzle==2.0.0 +et_xmlfile==2.0.0 +filelock==3.16.1 +fonttools==4.55.3 +future==1.0.0 +gwcs==0.22.1 +idna==3.10 +imageio==2.36.1 +importlib_metadata==8.5.0 +iniconfig==2.0.0 +jmespath==1.0.1 +jplephem==2.22 +jsonschema==4.23.0 +jsonschema-specifications==2024.10.1 +kiwisolver==1.4.7 +lazy_loader==0.4 +lxml==5.3.0 +matplotlib==3.10.0 +networkx==3.4.2 +numpy==1.26.4 +opencv-python-headless==4.10.0.84 +openpyxl==3.1.5 +packaging==24.2 +Parsley==1.3 +photutils==2.0.2 +pillow==11.0.0 +pluggy==1.5.0 +poppy==1.1.1 +pyerfa==2.0.1.5 +pyparsing==3.2.0 +pysiaf==0.24.1 +pytest==8.3.4 +pytest-cov==6.0.0 +pytest-doctestplus==1.3.0 +python-dateutil==2.9.0.post0 +PyYAML==6.0.2 +readchar==4.2.1 +referencing==0.35.1 +requests==2.32.3 +requests-mock==1.12.1 +rpds-py==0.22.3 +ruff==0.8.4 +scikit-image==0.25.0 +scipy==1.14.1 +semantic-version==2.10.0 +setuptools==75.1.0 +six==1.17.0 +spherical_geometry==1.3.2 +stcal==1.11.0 +stdatamodels==2.2.0 +stpipe==0.8.0 +stsci.imagestats==1.8.3 +stsci.stimage==0.2.9 +synphot==1.5.0 +tifffile==2024.12.12 +tweakwcs==0.8.9 +urllib3==2.2.3 +wheel==0.44.0 +wiimatch==0.3.2 +zipp==3.21.0 From ec3304909d54f7c72ad36ed74366267fe8839ed4 Mon Sep 17 00:00:00 2001 From: Tyler Pauly Date: Sat, 21 Dec 2024 09:31:22 -0500 Subject: [PATCH 5/5] Prepare for post-1.17.0 development (#9025) --- requirements-sdp.txt | 78 -------------------------------------------- 1 file changed, 78 deletions(-) diff --git a/requirements-sdp.txt b/requirements-sdp.txt index f6f1a9c478..a110a4b011 100644 --- a/requirements-sdp.txt +++ b/requirements-sdp.txt @@ -14,81 +14,3 @@ # conda activate sdp # pip install -e .[test,sdp] # pip freeze | grep -v jwst >> requirements-sdp.txt -asdf==4.0.0 -asdf-astropy==0.7.0 -asdf_coordinates_schemas==0.3.0 -asdf_standard==1.1.1 -asdf_transform_schemas==0.5.0 -asdf_wcs_schemas==0.4.0 -astropy==7.0.0 -astropy-iers-data==0.2024.12.16.0.35.48 -attrs==24.3.0 -BayesicFitting==3.2.3 -certifi==2024.12.14 -charset-normalizer==3.4.0 -ci_watson==0.8.0 -colorama==0.4.6 -contourpy==1.3.1 -coverage==7.6.9 -crds==12.0.8 -cycler==0.12.1 -drizzle==2.0.0 -et_xmlfile==2.0.0 -filelock==3.16.1 -fonttools==4.55.3 -future==1.0.0 -gwcs==0.22.1 -idna==3.10 -imageio==2.36.1 -importlib_metadata==8.5.0 -iniconfig==2.0.0 -jmespath==1.0.1 -jplephem==2.22 -jsonschema==4.23.0 -jsonschema-specifications==2024.10.1 -kiwisolver==1.4.7 -lazy_loader==0.4 -lxml==5.3.0 -matplotlib==3.10.0 -networkx==3.4.2 -numpy==1.26.4 -opencv-python-headless==4.10.0.84 -openpyxl==3.1.5 -packaging==24.2 -Parsley==1.3 -photutils==2.0.2 -pillow==11.0.0 -pluggy==1.5.0 -poppy==1.1.1 -pyerfa==2.0.1.5 -pyparsing==3.2.0 -pysiaf==0.24.1 -pytest==8.3.4 -pytest-cov==6.0.0 -pytest-doctestplus==1.3.0 -python-dateutil==2.9.0.post0 -PyYAML==6.0.2 -readchar==4.2.1 -referencing==0.35.1 -requests==2.32.3 -requests-mock==1.12.1 -rpds-py==0.22.3 -ruff==0.8.4 -scikit-image==0.25.0 -scipy==1.14.1 -semantic-version==2.10.0 -setuptools==75.1.0 -six==1.17.0 -spherical_geometry==1.3.2 -stcal==1.11.0 -stdatamodels==2.2.0 -stpipe==0.8.0 -stsci.imagestats==1.8.3 -stsci.stimage==0.2.9 -synphot==1.5.0 -tifffile==2024.12.12 -tweakwcs==0.8.9 -urllib3==2.2.3 -wheel==0.44.0 -wiimatch==0.3.2 -zipp==3.21.0