Merge branch 'main' into py313

.github/workflows/build.yml

@@ -16,7 +16,7 @@ jobs:
     name: check that `requirements-sdp.txt` is populated
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871  # v4.2.1
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2
       - run: grep -v '^ *#' requirements-sdp.txt
   build:
     needs: [ check ]

.github/workflows/changelog.yml

@@ -18,10 +18,10 @@ jobs:
     if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-changelog-entry-needed') }}
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3  # v5.2.0
+      - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b  # v5.3.0
         with:
           python-version: 3
-      - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871  # v4.2.1
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2
         with:
           fetch-depth: 0
       - run: pip install .
@@ -32,7 +32,7 @@ jobs:
     if: ${{ !contains(github.event.pull_request.labels.*.name, 'allow-manual-changelog-edit') }}
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871  # v4.2.1
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2
         with:
           fetch-depth: 0
       - name: prevent direct changes to `CHANGES.rst`

.github/workflows/ci.yml

@@ -34,17 +34,20 @@ jobs:
   pre-commit:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871  # v4.2.1
-    - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3  # v5.2.0
+    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2
+    - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b  # v5.3.0
+      with:
+        python-version: '3.12'
     - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd  # v3.0.1
   check:
     uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@924441154cf3053034c6513d5e06c69d262fb9a6  # v1.13.0
     with:
       default_python: "3.12"
       envs: |
         - linux: check-dependencies
+        - linux: check-types
   latest_crds_contexts:
-    uses: spacetelescope/crds/.github/workflows/contexts.yml@94138b4501c9487535fd6b977492fc1a2c319708  # 12.0.2
+    uses: ./.github/workflows/contexts.yml
   crds_context:
     needs: [ latest_crds_contexts ]
     runs-on: ubuntu-latest

.github/workflows/ci_cron.yml

@@ -23,7 +23,7 @@ concurrency:
 
 jobs:
   latest_crds_contexts:
-    uses: spacetelescope/crds/.github/workflows/contexts.yml@94138b4501c9487535fd6b977492fc1a2c319708  # 12.0.2
+    uses: ./.github/workflows/contexts.yml
   crds_context:
     needs: [ latest_crds_contexts ]
     runs-on: ubuntu-latest

.github/workflows/contexts.yml

@@ -0,0 +1,26 @@
+name: contexts
+
+on:
+  workflow_call:
+    outputs:
+      jwst:
+        value: ${{ jobs.contexts.outputs.jwst }}
+  workflow_dispatch:
+
+jobs:
+  contexts:
+    name: retrieve latest CRDS contexts
+    runs-on: ubuntu-latest
+    outputs:
+      jwst: ${{ steps.jwst_crds_context.outputs.pmap }}
+    steps:
+      - id: jwst_crds_context
+        env:
+          OBSERVATORY: jwst
+          CRDS_SERVER_URL: https://jwst-crds.stsci.edu
+        run: >
+          echo "pmap=$(
+          curl -s -X POST -d '{"jsonrpc": "1.0", "method": "get_default_context", "params": ["${{ env.OBSERVATORY }}", null], "id": 1}' ${{ env.CRDS_SERVER_URL }}/json/ --retry 8 --connect-timeout 10 |
+          python -c "import sys, json; print(json.load(sys.stdin)['result'])"
+          )" >> $GITHUB_OUTPUT
+      - run: if [[ ! -z "${{ steps.jwst_crds_context.outputs.pmap }}" ]]; then echo ${{ steps.jwst_crds_context.outputs.pmap }}; else exit 1; fi

.github/workflows/tests_devdeps.yml

@@ -32,7 +32,7 @@ concurrency:
 
 jobs:
   latest_crds_contexts:
-    uses: spacetelescope/crds/.github/workflows/contexts.yml@94138b4501c9487535fd6b977492fc1a2c319708  # 12.0.2
+    uses: ./.github/workflows/contexts.yml
   crds_context:
     needs: [ latest_crds_contexts ]
     runs-on: ubuntu-latest

CHANGES.rst

@@ -1,3 +1,12 @@
+1.16.1 (2024-10-30)
+===================
+
+resample_spec
+-------------
+
+- Update NIRSpec spectral resampling to add a missing correction factor in resampled
+  WCS tangent plane transformation. [#8908]
+
 1.16.0 (2024-09-20)
 ===================
 

README.md

@@ -216,12 +216,13 @@ the specified context and less than the context for the next release.
 
 | jwst tag            | DMS build | SDP_VER  | CRDS_CONTEXT | Released   | Ops Install | Notes                                         |
 |---------------------|-----------|----------|--------------|------------|-------------|-----------------------------------------------|
-| 1.16.0              | B11.1rc1  | TBD      | 1281         | 2024-09-20 | TBD         | First release candidate for B11.1             |
-| 1.15.1              | B11.0     | 2024.2.2 | 1242         | 2024-07-08 | 2024-09-12  | Final release candidate for B11.0             |
-| 1.15.0              | B11.0rc1  |          | 1241         | 2024-06-26 |             | First release candidate for B11.0             |
-| 1.14.1              |           |          | 1238         | 2024-06-27 |             | PyPI-only release for external users          |
-| 1.14.0              | B10.2.1   | 2024.1.1 | 1238         | 2024-03-29 | 2024-06-12  | Final release candidate for B10.2.1           |
-| 1.13.4              |           |          | 1185         | 2024-01-25 |             | PyPI-only release for external users          |
+| 1.16.1              | B11.1.1   | 2024.3.1 | 1298         | 2024-11-13 | TBD         | Final release candidate for B11.1             |
+| 1.16.0              | B11.1     | 2024.3.0 | 1298         | 2024-09-20 | TBD         | First release candidate for B11.1             |
+| 1.15.1              | B11.0     | 2024.2.2 | 1293         | 2024-07-08 | 2024-09-12  | Final release candidate for B11.0             |
+| 1.15.0              | B11.0rc1  |          | 1274         | 2024-06-26 |             | First release candidate for B11.0             |
+| 1.14.1              |           |          | 1240         | 2024-06-27 |             | PyPI-only release for external users          |
+| 1.14.0              | B10.2.1   | 2024.1.1 | 1240         | 2024-03-29 | 2024-06-12  | Final release candidate for B10.2.1           |
+| 1.13.4              |           |          | 1210         | 2024-01-25 |             | PyPI-only release for external users          |
 | 1.13.3              | B10.1     | 2023.4.0 | 1181         | 2024-01-05 |             | Final release candidate for B10.1             |
 | 1.13.2              | B10.1rc3  | 2023.4.0 | 1181         | 2023-12-21 |             | Third release candidate for B10.1             |
 | 1.13.1              | B10.1rc2  | 2023.4.0 | 1181         | 2023-12-19 |             | Second release candidate for B10.1            |

changes/8828.outlier_detection.rst

@@ -0,0 +1 @@
+For slit spectra, threshold outliers with a median error across exposures instead of input error from the exposure itself.

changes/8831.datamodels.rst

@@ -0,0 +1 @@
+Remove memory-saving options from ModelContainer

changes/8852.general.rst

@@ -0,0 +1 @@
+Added mypy type checking to CI checks

changes/8866.resample.rst

@@ -0,0 +1 @@
+Updated resample code to support the new ``drizzle`` API, see https://github.com/spacetelescope/drizzle/pull/134 for more details.

changes/8870.outlier_detection.rst

@@ -0,0 +1 @@
+Remove deprecated nlow and nhigh parameters from outlier detection step.

changes/8874.assign_wcs.rst

@@ -0,0 +1 @@
+Catch NaN values in msa tables for source positions in slit and replace with slit center.

changes/8880.outlier_detection.rst

@@ -0,0 +1 @@
+reorganize outlier detection documentation

changes/8885.general.rst

@@ -0,0 +1 @@
+Give regtest okify results unique subdirectories.

changes/8892.skymatch.rst

@@ -0,0 +1 @@
+Resolve warnings emitted by NumPy 2 when running skymatch.

changes/8893.resample.rst

@@ -0,0 +1 @@
+Use s_region list to calculate output footprint instead of re-computing via WCS transforms

changes/8897.assign_wcs.rst

@@ -0,0 +1 @@
+Use pixel vertices to define s_region instead of pixel centers."

changes/8907.pipeline.rst

@@ -0,0 +1 @@
+Fixed a bug leading to incorrect area extensions, and sometimes crashes, in the coron3 pipeline

changes/8908.resample_spec.rst

@@ -0,0 +1 @@
+Update NIRSpec spectral resampling to add a missing correction factor in resampled WCS tangent plane transformation.

changes/8909.scripts.rst

@@ -0,0 +1 @@
+Remove the outdated schema_editor script.

changes/8911.cube_build.rst

@@ -0,0 +1 @@
+For moving-target IFU data, set RA, Dec header information of s3d products according to the mean of input models instead of the first input model.

changes/8913.cube_build.rst

@@ -0,0 +1 @@
+Tweak the cube_build spaxel debugging option to provide filename info and match outputs to the stated ordering.

changes/8916.background.rst

@@ -0,0 +1 @@
+Apply bitwise operations in correct order when counting good pixels in the background mask during WFSS background subtraction.

changes/8918.datamodels.rst

@@ -0,0 +1 @@
+Update ModelLibrary to use meta.asn.exptype instead of meta.exptype.

changes/8926.docs.rst

@@ -0,0 +1 @@
+Mention possible need to provide package name to strun when using aliases.

changes/8927.combine_1d.rst

@@ -0,0 +1 @@
+Fix wavelength sort order for single input spectrum.

changes/8927.master_background.rst

@@ -0,0 +1 @@
+Fix ModelContainer handling for user background input.

changes/8927.pipeline.rst

@@ -0,0 +1 @@
+Add 'mbsub' to the list of known suffixes, for ``master_background`` correction in ``calwebb_spec3``.

changes/8932.master_background.rst

@@ -0,0 +1 @@
+Added pixel-to-pixel sigma clipping on input backgrounds in the NIRSpec MOS master_background_mos step.  Added option to median filter master background spectrum in both the master_background and master_background_mos steps.

changes/8935.general.rst

@@ -0,0 +1 @@
+Include xml and db test data as package data for lib tests.

changes/8945.stpipe.rst

@@ -0,0 +1 @@
+Remove all uses of Step.__call__ to allow it's deprecation.

changes/8947.pipeline.rst

@@ -0,0 +1 @@
+Transfer wcsinfo metadata to new MultiSlitModel created during ``calwebb_spec2`` processing of NIRSpec MSA data.

changes/8950.general.rst

@@ -0,0 +1 @@
+Update minimum required version of crds to allow for "data release style" contexts.

changes/8952.firstframe.rst

@@ -0,0 +1 @@
+Update the firstframe step to optionally not flag when a ramp saturates in group 3

changes/8958.general.rst

@@ -0,0 +1 @@
+When blending metadata don't store columns containing all missing value (nans).

changes/8975.resample.rst

@@ -0,0 +1 @@
+Removed allowed_memory parameter and DMODEL_ALLOWED_MEMORY environment variable

docs/jwst/combine_1d/arguments.rst

@@ -1,7 +1,7 @@
 Step Arguments
 ==============
 
-The ``combine_1d`` step has one step-specific argument:
+The ``combine_1d`` step has two step-specific arguments:
 
 ``--exptime_key``
   This is a case-insensitive string that identifies the metadata element
@@ -13,3 +13,10 @@ The ``combine_1d`` step has one step-specific argument:
   the string is "unit_weight" or "unit weight", the same weight (1) will
   be used for all input spectra.  If the string is anything else, a warning
   will be logged and unit weight will be used.
+
+``--sigma_clip``
+  Optional factor for sigma clipping outliers when combining spectra. If
+  a floating point value is provided for ``sigma_clip``, this value will be
+  used to set an outlier threshold for any pixels in the input spectra that 
+  deviate from the median and median absolute deviation of the inputs.
+  Defaults to None (such that no clipping is performed).

docs/jwst/dark_current/description.rst

@@ -6,8 +6,7 @@ Description
 
 Assumptions
 -----------
-It is assumed that the input science data have *NOT* had the zero group (or
-bias) subtracted. We also do not want the dark subtraction process to remove
+We do not want the dark subtraction process to remove
 the bias signal from the science exposure, therefore the dark reference data
 should have their own group zero subtracted from all groups. This means that
 group zero of the dark reference data will effectively be zero-valued.

docs/jwst/firstframe/arguments.rst

@@ -1,4 +1,13 @@
 Step Arguments
 ==============
 
-The first frame correction has no step-specific arguments.
+The ``firstframe`` step has the following step-specific arguments.
+
+``--bright_use_group1`` (boolean, default=False)
+    If True, setting the group 1 groupdq to DO_NOT_USE will not be done 
+    for pixels that have the saturation flag set for group 3.  
+    This will allow a slope to be determined for pixels that saturate in group 3.
+    This change in flagging will only impact pixels that saturate in group 3, the behavior
+    for all other pixels will be unchanged.
+    The `bright_use_group1` flag can be set for all data, only data/pixels that saturate 
+    in group 3 will see a difference in behavior.

docs/jwst/master_background/arguments.rst

@@ -4,6 +4,12 @@ Step Arguments
 ==============
 The master background subtraction step uses the following optional arguments.
 
+``--median_kernel``
+  Optional user-supplied kernel size for a moving-median boxcar filter to filter 
+  outliers in the master background spectrum.  The kernel size must be an odd integer.
+  Even integers will be rounded down to the nearest odd integer.
+  Defaults to 1 (which applies no median filtering).
+
 ``--user_background``
   The file name of a user-supplied 1-D master background spectrum. Must be in the form
   of a standard :ref:`x1d <x1d>` product containing a single 'EXTRACT1D' extension.
@@ -15,6 +21,10 @@ The master background subtraction step uses the following optional arguments.
 ``--save_background``
   A boolean indicating whether the computed 1-D master background spectrum should be saved
   to a file. The file name uses a product type suffix of "masterbg".
+  For the `master_background_mos` step, multiple files will be produced including the 1-D 
+  master background spectrum (saved with the suffix "masterbg1d"), the expanded 2-D background spectra
+  for each MOS slitlet (with the suffix "masterbg2d"), and the 1-D background spectra 
+  that were combined into the master background spectrum (with the suffix "bkgx1d").
   If a user-supplied background is specified, this argument is ignored.
   Defaults to ``False``.
 
@@ -28,3 +38,13 @@ The master background subtraction step uses the following optional arguments.
 ``--output_use_model``
   A boolean indicating whether to use the "filename" meta attribute in the data model to
   determine the name of the output file created by the step. Defaults to ``True``.
+
+``--sigma_clip``
+  Factor for sigma clipping outliers and contaminated spectra when combining MOS 
+  background spectra in the `master_background_mos` step.  The value of ``sigma_clip`` 
+  will be used to set an outlier threshold for clipping any pixels in the background 
+  spectra that deviate from the median and median absolute deviation of the inputs before
+  combining the background spectra.  Setting ``sigma_clip`` to None will
+  skip any outlier clipping.  This parameter is only available in `master_background_mos`
+  step and is not available in the generic `master_background` step.
+  Defaults to 3.