Merge branch 'main' into jp3593_take2

.github/pull_request_template.md

@@ -12,14 +12,19 @@ Closes #
 
 This PR addresses ...
 
-**Checklist**
-
-- [ ] for a public change, added a towncrier news fragment in `changes/`: <details><summary>`echo "changed something" > changes/<PR#>.<changetype>.rst`</summary>
-
-    - ``changes/<PR#>.apichange.rst``: change to public API
-    - ``changes/<PR#>.bugfix.rst``: fixes an issue
-    - ``changes/<PR#>.general.rst``: infrastructure or miscellaneous change
-- [ ] updated relevant tests
-- [ ] updated relevant documentation
-- [ ] updated relevant milestone(s)
-- [ ] added relevant label(s)
+<!-- if you can't perform these tasks due to permissions, please ask a maintainer to do them -->
+## Tasks
+- [ ] update or add relevant tests
+- [ ] update relevant docstrings and / or `docs/` page
+- [ ] Does this PR change any API used downstream? (if not, label with `no-changelog-entry-needed`)
+  - [ ] write news fragment(s) in `changes/`: `echo "changed something" > changes/<PR#>.<changetype>.rst` (see below for change types)
+  - [ ] run regression tests with this branch installed (`"git+https://github.com/<fork>/stcal@<branch>"`)
+    - [ ] [`jwst` regression test](https://github.com/spacetelescope/RegressionTests/actions/workflows/jwst.yml) 
+    - [ ] [`romancal` regression test](https://github.com/spacetelescope/RegressionTests/actions/workflows/romancal.yml)
+
+<details><summary>news fragment change types...</summary>
+
+- ``changes/<PR#>.apichange.rst``: change to public API
+- ``changes/<PR#>.bugfix.rst``: fixes an issue
+- ``changes/<PR#>.general.rst``: infrastructure or miscellaneous change
+</details>

.gitignore

@@ -4,6 +4,9 @@ __pycache__/
 *$py.class
 *~
 
+# Temp files
+*.*.swp
+
 # C extensions
 *.so
 

CHANGES.rst

@@ -1,3 +1,42 @@
+1.8.2 (2024-09-10)
+==================
+
+Bug Fixes
+---------
+
+- Implement byteorder swap method that is forward-compatible with numpy 2.0 in
+  jwst ramp_fitting. (`#282
+  <https://github.com/spacetelescope/stcal/issues/282>`_)
+- [jump] Fix a logical bug in the jump step for usage of > vs >= per JP-3689.
+  (`#285 <https://github.com/spacetelescope/stcal/issues/285>`_)
+
+
+General
+-------
+
+- [ramp_fitting] Moving the read noise recalculation due to CHARGELOSS flagging
+  from
+  the JWST ramp fit step code into the STCAL ramp fit C-extension. (`#275
+  <https://github.com/spacetelescope/stcal/issues/275>`_)
+
+
+1.8.1 (2024-09-08)
+==================
+
+Bug Fixes
+---------
+
+- Fixed memory leak in C-extension. (`#281
+  <https://github.com/spacetelescope/stcal/issues/281>`_)
+
+
+General
+-------
+
+- use ``towncrier`` to handle changelog entries (`#284
+  <https://github.com/spacetelescope/stcal/issues/284>`_)
+
+
 1.8.0 (2024-08-14)
 ==================
 
@@ -6,6 +45,12 @@ General
 
 - Add TweakReg submodule. [#267]
 
+ramp_fitting
+~~~~~~~~~~~~
+
+- Move the CHARGELOSS read noise variance recalculation from the JWST step
+  code to the C extension to simplify the code and improve performance.[#275]
+
 Changes to API
 --------------
 

changes/286.general.1.rst

@@ -0,0 +1 @@
+Improve handling of catalog web service connectivity issues.

changes/286.general.rst

@@ -0,0 +1 @@
+Add clip_accum parameter to alignment methods.

src/stcal/jump/jump.py

@@ -972,7 +972,7 @@ def find_faint_extended(
 
     first_diffs_masked = np.ma.masked_array(first_diffs, mask=np.isnan(first_diffs))
     warnings.filterwarnings("ignore")
-    if nints > minimum_sigclip_groups:
+    if nints >= minimum_sigclip_groups:
         mean, median, stddev = stats.sigma_clipped_stats(first_diffs_masked, sigma=5, axis=0)
     else:
         median_diffs = np.nanmedian(first_diffs_masked, axis=(0, 1))

src/stcal/ramp_fitting/ols_fit.py

@@ -687,9 +687,9 @@ def ols_ramp_fit_single(ramp_data, buffsize, save_opt, readnoise_2d, gain_2d, we
         # ramp fitting.
         rn_bswap, gain_bswap = bswap
         if rn_bswap:
-            readnoise_2d.newbyteorder('S').byteswap(inplace=True)
+            readnoise_2d.view(readnoise_2d.dtype.newbyteorder('S')).byteswap(inplace=True)
         if gain_bswap:
-            gain_2d.newbyteorder('S').byteswap(inplace=True)
+            gain_2d.view(gain_2d.dtype.newbyteorder('S')).byteswap(inplace=True)
 
         c_diff = c_end - c_start
         log.info(f"Ramp Fitting C Time: {c_diff}")
@@ -731,7 +731,7 @@ def handle_array_endianness(arr, sys_order):
     arr_order = arr.dtype.byteorder
     bswap = False
     if (arr_order == ">" and sys_order == "<") or (arr_order == "<" and sys_order == ">"):
-        arr.newbyteorder('S').byteswap(inplace=True)
+        arr.view(arr.dtype.newbyteorder('S')).byteswap(inplace=True)
         bswap = True
 
     return arr, bswap
@@ -920,6 +920,7 @@ def discard_miri_groups(ramp_data):
     data = ramp_data.data
     err = ramp_data.err
     groupdq = ramp_data.groupdq
+    orig_gdq = ramp_data.orig_gdq
 
     n_int, ngroups, nrows, ncols = data.shape
 
@@ -949,6 +950,8 @@ def discard_miri_groups(ramp_data):
     if num_bad_slices > 0:
         data = data[:, num_bad_slices:, :, :]
         err = err[:, num_bad_slices:, :, :]
+        if orig_gdq is not None:
+            orig_gdq = orig_gdq[:, num_bad_slices:, :, :]
 
     log.info("Number of leading groups that are flagged as DO_NOT_USE: %s", num_bad_slices)
 
@@ -968,6 +971,8 @@ def discard_miri_groups(ramp_data):
         data = data[:, :-1, :, :]
         err = err[:, :-1, :, :]
         groupdq = groupdq[:, :-1, :, :]
+        if orig_gdq is not None:
+            orig_gdq = orig_gdq[:, :-1, :, :]
 
         log.info("MIRI dataset has all pixels in the final group flagged as DO_NOT_USE.")
 
@@ -981,6 +986,8 @@ def discard_miri_groups(ramp_data):
     ramp_data.data = data
     ramp_data.err = err
     ramp_data.groupdq = groupdq
+    if orig_gdq is not None:
+        ramp_data.orig_gdq = orig_gdq
 
     return True
 

src/stcal/ramp_fitting/ramp_fit.py

@@ -30,7 +30,7 @@
 BUFSIZE = 1024 * 300000  # 300Mb cache size for data section
 
 
-def create_ramp_fit_class(model, dqflags=None, suppress_one_group=False):
+def create_ramp_fit_class(model, algorithm, dqflags=None, suppress_one_group=False):
     """
     Create an internal ramp fit class from a data model.
 
@@ -58,11 +58,24 @@ def create_ramp_fit_class(model, dqflags=None, suppress_one_group=False):
     else:
         dark_current_array = model.average_dark_current
 
+    orig_gdq = None
+    if algorithm.upper() == "OLS_C":
+        wh_chargeloss = np.where(np.bitwise_and(model.groupdq.astype(np.uint32), dqflags['CHARGELOSS']))
+        if len(wh_chargeloss[0]) > 0:
+            orig_gdq = model.groupdq.copy()
+        del wh_chargeloss
+
     if isinstance(model.data, u.Quantity):
         ramp_data.set_arrays(model.data.value, model.err.value, model.groupdq,
                              model.pixeldq, dark_current_array)
     else:
-        ramp_data.set_arrays(model.data, model.err, model.groupdq, model.pixeldq, dark_current_array)
+        ramp_data.set_arrays(
+            model.data,
+            model.err,
+            model.groupdq,
+            model.pixeldq,
+            dark_current_array,
+            orig_gdq)
 
     # Attribute may not be supported by all pipelines.  Default is NoneType.
     drop_frames1 = model.meta.exposure.drop_frames1 if hasattr(model, "drop_frames1") else None
@@ -78,6 +91,7 @@ def create_ramp_fit_class(model, dqflags=None, suppress_one_group=False):
     if "zero_frame" in model.meta.exposure and model.meta.exposure.zero_frame:
         ramp_data.zeroframe = model.zeroframe
 
+    ramp_data.algorithm = algorithm
     ramp_data.set_dqflags(dqflags)
     ramp_data.start_row = 0
     ramp_data.num_rows = ramp_data.data.shape[2]
@@ -170,7 +184,7 @@ def ramp_fit(
     # Create an instance of the internal ramp class, using only values needed
     # for ramp fitting from the to remove further ramp fitting dependence on
     # data models.
-    ramp_data = create_ramp_fit_class(model, dqflags, suppress_one_group)
+    ramp_data = create_ramp_fit_class(model, algorithm, dqflags, suppress_one_group)
 
     if algorithm.upper() == "OLS_C":
         ramp_data.run_c_code = True

src/stcal/ramp_fitting/ramp_fit_class.py

@@ -10,6 +10,10 @@ def __init__(self):
         self.pixeldq = None
         self.average_dark_current = None
 
+        # Needed for CHARGELOSS recomputation
+        self.orig_gdq = None
+        self.algorithm = None
+
         # Meta information
         self.instrument_name = None
 
@@ -25,6 +29,7 @@ def __init__(self):
         self.flags_saturated = None
         self.flags_no_gain_val = None
         self.flags_unreliable_slope = None
+        self.flags_chargeloss = None
 
         # ZEROFRAME
         self.zframe_mat = None
@@ -41,13 +46,15 @@ def __init__(self):
 
         # C code debugging switch.
         self.run_c_code = False
+        self.run_chargeloss = True
+        # self.run_chargeloss = False
 
         self.one_groups_locs = None  # One good group locations.
         self.one_groups_time = None  # Time to use for one good group ramps.
 
         self.current_integ = -1
 
-    def set_arrays(self, data, err, groupdq, pixeldq, average_dark_current):
+    def set_arrays(self, data, err, groupdq, pixeldq, average_dark_current, orig_gdq=None):
         """
         Set the arrays needed for ramp fitting.
 
@@ -72,6 +79,11 @@ def set_arrays(self, data, err, groupdq, pixeldq, average_dark_current):
         average_dark_current : ndarray (float32)
             2-D array containing the average dark current. It has
             dimensions (nrows, ncols)
+
+        orig_gdq : ndarray
+            4-D array containing a copy of the original group DQ array.  Since
+            the group DQ array can be modified during ramp fitting, this keeps
+            around the original group DQ flags passed to ramp fitting.
         """
         # Get arrays from the data model
         self.data = data
@@ -80,6 +92,8 @@ def set_arrays(self, data, err, groupdq, pixeldq, average_dark_current):
         self.pixeldq = pixeldq
         self.average_dark_current = average_dark_current
 
+        self.orig_gdq = orig_gdq
+
     def set_meta(self, name, frame_time, group_time, groupgap, nframes, drop_frames1=None):
         """
         Set the metainformation needed for ramp fitting.
@@ -131,6 +145,8 @@ def set_dqflags(self, dqflags):
         self.flags_saturated = dqflags["SATURATED"]
         self.flags_no_gain_val = dqflags["NO_GAIN_VALUE"]
         self.flags_unreliable_slope = dqflags["UNRELIABLE_SLOPE"]
+        if self.algorithm is not None and self.algorithm.upper() == "OLS_C":
+            self.flags_chargeloss = dqflags["CHARGELOSS"]
 
     def dbg_print_types(self):
         # Arrays from the data model
@@ -200,6 +216,16 @@ def dbg_print_pixel_info(self, row, col):
         # print(f"    err :\n{self.err[:, :, row, col]}")
         # print(f"    pixeldq :\n{self.pixeldq[row, col]}")
 
+    def dbg_print_info(self):
+        print(" ")
+        nints, ngroups, nrows, ncols = self.data.shape
+        for row in range(nrows):
+            for col in range(ncols):
+                print("=" * 80)
+                print(f"**** Pixel ({row}, {col}) ****")
+                self.dbg_print_pixel_info(row, col)
+        print("=" * 80)
+
     def dbg_write_ramp_data_pix_pre(self, fname, row, col, fd):
         fd.write("def create_ramp_data_pixel():\n")
         indent = INDENT