From f9d2fa9af08ea9d1e1deb3b0f06363d5bbb30a26 Mon Sep 17 00:00:00 2001 From: Ken MacDonald Date: Thu, 8 Jun 2023 06:49:28 -0400 Subject: [PATCH 01/13] Removing some usage of effintim and replacing it with group_time. --- src/stcal/ramp_fitting/ols_fit.py | 11 +++++++++-- src/stcal/ramp_fitting/utils.py | 21 +++++++++++++-------- tests/test_ramp_fitting.py | 30 +++++++++++++++++++++++------- 3 files changed, 45 insertions(+), 17 deletions(-) diff --git a/src/stcal/ramp_fitting/ols_fit.py b/src/stcal/ramp_fitting/ols_fit.py index 389ba0ec..8d98ce83 100644 --- a/src/stcal/ramp_fitting/ols_fit.py +++ b/src/stcal/ramp_fitting/ols_fit.py @@ -11,6 +11,13 @@ from . import utils +################## DEBUG ################## +# HELP!! +import sys +sys.path.insert(1, "/Users/kmacdonald/code/common") +from general_funcs import dbg_print +################## DEBUG ################## + log = logging.getLogger(__name__) log.setLevel(logging.DEBUG) @@ -1375,7 +1382,7 @@ def ramp_fit_overall( opt_res.var_p_seg = var_p4[:, :f_max_seg, :, :] opt_res.var_r_seg = var_r4[:, :f_max_seg, :, :] - opt_info = opt_res.output_optional(effintim) + opt_info = opt_res.output_optional(ramp_data.group_time) else: opt_info = None @@ -1396,7 +1403,7 @@ def ramp_fit_overall( # Output integration-specific results to separate file integ_info = utils.output_integ( - ramp_data, slope_int, dq_int, effintim, var_p3, var_r3, var_both3) + ramp_data, slope_int, dq_int, var_p3, var_r3, var_both3) if opt_res is not None: del opt_res diff --git a/src/stcal/ramp_fitting/utils.py b/src/stcal/ramp_fitting/utils.py index ecef1a3a..8d53c33a 100644 --- a/src/stcal/ramp_fitting/utils.py +++ b/src/stcal/ramp_fitting/utils.py @@ -7,6 +7,14 @@ import warnings +################## DEBUG ################## +# HELP!! +import sys +sys.path.insert(1, "/Users/kmacdonald/code/common") +from general_funcs import dbg_print +################## DEBUG ################## + + log = logging.getLogger(__name__) log.setLevel(logging.DEBUG) @@ -250,7 +258,7 @@ def shrink_crmag(self, n_int, dq_cube, imshape, nreads, jump_det): else: self.cr_mag_seg = cr_com[:, :max_num_crs, :, :] - def output_optional(self, effintim): + def output_optional(self, group_time): """ These results are the cosmic ray magnitudes in the segment-specific results for the count rates, y-intercept, @@ -265,7 +273,7 @@ def output_optional(self, effintim): Parameters ---------- - effintim : float + group_time : float effective integration time for a single group Returns @@ -284,7 +292,7 @@ def output_optional(self, effintim): self.weights[1. / self.weights > LARGE_VARIANCE_THRESHOLD] = 0. warnings.resetwarnings() - self.slope_seg /= effintim + self.slope_seg /= group_time opt_info = (self.slope_seg, self.sigslope_seg, self.var_p_seg, self.var_r_seg, self.yint_seg, self.sigyint_seg, @@ -681,7 +689,7 @@ def calc_pedestal(ramp_data, num_int, slope_int, firstf_int, dq_first, nframes, return ped -def output_integ(ramp_data, slope_int, dq_int, effintim, var_p3, var_r3, var_both3): +def output_integ(ramp_data, slope_int, dq_int, var_p3, var_r3, var_both3): """ For the OLS algorithm, construct the output integration-specific results. Any variance values that are a large fraction of the default value @@ -702,9 +710,6 @@ def output_integ(ramp_data, slope_int, dq_int, effintim, var_p3, var_r3, var_bot dq_int : ndarray Data cube of DQ arrays for each integration, 3-D int - effintim : float - Effective integration time per integration - var_p3 : ndarray Cube of integration-specific values for the slope variance due to Poisson noise only, 3-D float @@ -731,7 +736,7 @@ def output_integ(ramp_data, slope_int, dq_int, effintim, var_p3, var_r3, var_bot var_r3[var_r3 > LARGE_VARIANCE_THRESHOLD] = 0. var_both3[var_both3 > LARGE_VARIANCE_THRESHOLD] = 0. - data = slope_int / effintim + data = slope_int / ramp_data.group_time invalid_data = ramp_data.flags_saturated | ramp_data.flags_do_not_use data[np.bitwise_and(dq_int, invalid_data).astype(bool)] = np.nan diff --git a/tests/test_ramp_fitting.py b/tests/test_ramp_fitting.py index 73eb6300..adc5a565 100644 --- a/tests/test_ramp_fitting.py +++ b/tests/test_ramp_fitting.py @@ -4,6 +4,14 @@ from stcal.ramp_fitting.ramp_fit_class import RampData +################## DEBUG ################## +# HELP!! +import sys +sys.path.insert(1, "/Users/kmacdonald/code/common") +from general_funcs import dbg_print +################## DEBUG ################## + + DELIM = "-" * 70 # single group intergrations fail in the GLS fitting @@ -470,12 +478,20 @@ def run_one_group_ramp_suppression(nints, suppress): In the second integration all pixels have all good groups. """ + # XXX + # This is a bad way to do things. The inputs are bad. The + # group_time is defined independently from nframes, frame_time, + # and groupgap, which is bad. + # Define the data. ngroups, nrows, ncols = 5, 1, 3 dims = (nints, ngroups, nrows, ncols) rnoise, gain = 10, 1 - nframes, group_time, frame_time = 1, 5.0, 1 + # nframes, group_time, frame_time = 1, 5.0, 1 + nframes, frame_time, groupgap = 1, 1, 0 var = rnoise, gain + # group_time = (nframes + groupgap) * frame_time + group_time = 5.0 tm = nframes, group_time, frame_time # Using the above create the classes and arrays. @@ -546,7 +562,7 @@ def test_one_group_ramp_suppressed_one_integration(): # Check slopes information cdata, cdq, cvp, cvr, cerr = cube - check = np.array([[[np.nan, np.nan, 1.0000001]]]) + check = np.array([[[np.nan, np.nan, 0.20000002]]]) np.testing.assert_allclose(cdata, check, tol) check = np.array([[[DNU | SAT, DNU, GOOD]]]) @@ -591,7 +607,7 @@ def test_one_group_ramp_not_suppressed_one_integration(): # Check slopes information cdata, cdq, cvp, cvr, cerr = cube - check = np.array([[[np.nan, 1., 1.0000001]]]) + check = np.array([[[np.nan, 0.2, 0.20000002]]]) np.testing.assert_allclose(cdata, check, tol) check = np.array([[[DNU | SAT, GOOD, GOOD]]]) @@ -637,8 +653,8 @@ def test_one_group_ramp_suppressed_two_integrations(): # Check slopes information cdata, cdq, cvp, cvr, cerr = cube - check = np.array([[[np.nan, np.nan, 1.0000001]], - [[1.0000001, 1.0000001, 1.0000001]]]) + check = np.array([[[np.nan, np.nan, 0.20000002]], + [[0.20000002, 0.20000002, 0.20000002]]]) np.testing.assert_allclose(cdata, check, tol) check = np.array([[[DNU | SAT, DNU, GOOD]], @@ -688,8 +704,8 @@ def test_one_group_ramp_not_suppressed_two_integrations(): # Check slopes information cdata, cdq, cvp, cvr, cerr = cube - check = np.array([[[np.nan, 1., 1.0000001]], - [[1.0000001, 1.0000001, 1.0000001]]]) + check = np.array([[[np.nan, 0.2, 0.20000002]], + [[0.20000002, 0.20000002, 0.20000002]]]) np.testing.assert_allclose(cdata, check, tol) check = np.array([[[DNU | SAT, GOOD, GOOD]], From affb6e0c09bd160fceaadd649a20d63a61e9ec93 Mon Sep 17 00:00:00 2001 From: Ken MacDonald Date: Thu, 8 Jun 2023 07:17:51 -0400 Subject: [PATCH 02/13] Removing unnecessary 'effintim' variable, which is just the group time. --- src/stcal/ramp_fitting/ols_fit.py | 21 +++++---------------- src/stcal/ramp_fitting/utils.py | 1 + tests/test_ramp_fitting.py | 8 ++++---- 3 files changed, 10 insertions(+), 20 deletions(-) diff --git a/src/stcal/ramp_fitting/ols_fit.py b/src/stcal/ramp_fitting/ols_fit.py index 8d98ce83..caa02174 100644 --- a/src/stcal/ramp_fitting/ols_fit.py +++ b/src/stcal/ramp_fitting/ols_fit.py @@ -838,9 +838,6 @@ def ramp_fit_slopes(ramp_data, gain_2d, readnoise_2d, save_opt, weighting): gdq_cube_shape : ndarray Group DQ dimensions - effintim : float - effective integration time for a single group - f_max_seg : int Actual maximum number of segments over all groups and segments @@ -882,14 +879,6 @@ def ramp_fit_slopes(ramp_data, gain_2d, readnoise_2d, save_opt, weighting): imshape = (nrows, ncols) cubeshape = (ngroups,) + imshape - # Calculate effective integration time (once EFFINTIM has been populated - # and accessible, will use that instead), and other keywords that will - # needed if the pedestal calculation is requested. Note 'nframes' - # is the number of given by the NFRAMES keyword, and is the number of - # frames averaged on-board for a group, i.e., it does not include the - # groupgap. - effintim = (nframes + groupgap) * frame_time - # Get GROUP DQ and ERR arrays from input file gdq_cube = groupdq gdq_cube_shape = gdq_cube.shape @@ -1005,7 +994,7 @@ def ramp_fit_slopes(ramp_data, gain_2d, readnoise_2d, save_opt, weighting): ramp_data.groupdq = groupdq ramp_data.pixeldq = inpixeldq - return max_seg, gdq_cube_shape, effintim, f_max_seg, dq_int, num_seg_per_int,\ + return max_seg, gdq_cube_shape, f_max_seg, dq_int, num_seg_per_int,\ sat_0th_group_int, opt_res, pixeldq, inv_var, med_rates @@ -1090,7 +1079,7 @@ def ramp_fit_compute_variances(ramp_data, gain_2d, readnoise_2d, fit_slopes_ans) max_seg = fit_slopes_ans[0] num_seg_per_int = fit_slopes_ans[5] - med_rates = fit_slopes_ans[10] + med_rates = fit_slopes_ans[9] var_p3, var_r3, var_p4, var_r4, var_both4, var_both3, \ inv_var_both4, s_inv_var_p3, s_inv_var_r3, s_inv_var_both3, segs_4 = \ @@ -1290,8 +1279,8 @@ def ramp_fit_overall( imshape = (nrows, ncols) # Unpack intermediate computations from preious steps - max_seg, gdq_cube_shape, effintim, f_max_seg, dq_int, num_seg_per_int = fit_slopes_ans[:6] - sat_0th_group_int, opt_res, pixeldq, inv_var, med_rates = fit_slopes_ans[6:] + max_seg, gdq_cube_shape, f_max_seg, dq_int, num_seg_per_int = fit_slopes_ans[:5] + sat_0th_group_int, opt_res, pixeldq, inv_var, med_rates = fit_slopes_ans[5:] var_p3, var_r3, var_p4, var_r4, var_both4, var_both3 = variances_ans[:6] inv_var_both4, s_inv_var_p3, s_inv_var_r3, s_inv_var_both3 = variances_ans[6:] @@ -1416,7 +1405,7 @@ def ramp_fit_overall( # Divide slopes by total (summed over all integrations) effective # integration time to give count rates. - c_rates = slope_dataset2 / effintim + c_rates = slope_dataset2 / ramp_data.group_time # Compress all integration's dq arrays to create 2D PIXELDDQ array for # primary output diff --git a/src/stcal/ramp_fitting/utils.py b/src/stcal/ramp_fitting/utils.py index 8d53c33a..8b00daa5 100644 --- a/src/stcal/ramp_fitting/utils.py +++ b/src/stcal/ramp_fitting/utils.py @@ -840,6 +840,7 @@ def shift_z(a, off): def get_efftim_ped(ramp_data): """ + XXX - Work to remove this function. Calculate the effective integration time for a single group, and return the number of frames per group, and the number of frames dropped between groups. diff --git a/tests/test_ramp_fitting.py b/tests/test_ramp_fitting.py index adc5a565..853d33f5 100644 --- a/tests/test_ramp_fitting.py +++ b/tests/test_ramp_fitting.py @@ -544,7 +544,7 @@ def test_one_group_ramp_suppressed_one_integration(): # Check slopes information sdata, sdq, svp, svr, serr = slopes - check = np.array([[np.nan, np.nan, 1.0000002]]) + check = np.array([[np.nan, np.nan, 0.20000005]]) np.testing.assert_allclose(sdata, check, tol) check = np.array([[DNU | SAT, DNU, GOOD]]) @@ -589,7 +589,7 @@ def test_one_group_ramp_not_suppressed_one_integration(): # Check slopes information sdata, sdq, svp, svr, serr = slopes - check = np.array([[np.nan, 1., 1.0000002]]) + check = np.array([[np.nan, 0.2, 0.20000005]]) np.testing.assert_allclose(sdata, check, tol) check = np.array([[DNU | SAT, GOOD, GOOD]]) @@ -635,7 +635,7 @@ def test_one_group_ramp_suppressed_two_integrations(): # Check slopes information sdata, sdq, svp, svr, serr = slopes - check = np.array([[1.0000001, 1.0000001, 1.0000002]]) + check = np.array([[0.20000002, 0.20000005, 0.20000005]]) np.testing.assert_allclose(sdata, check, tol) check = np.array([[GOOD, GOOD, GOOD]]) @@ -686,7 +686,7 @@ def test_one_group_ramp_not_suppressed_two_integrations(): # Check slopes information sdata, sdq, svp, svr, serr = slopes - check = np.array([[1.0000001, 1.0000002, 1.0000002]]) + check = np.array([[0.20000002, 0.20000005, 0.20000005]]) np.testing.assert_allclose(sdata, check, tol) check = np.array([[GOOD, GOOD, GOOD]]) From 14f8e8648a7d76861fea7e558abcdfcc73c66ff7 Mon Sep 17 00:00:00 2001 From: Ken MacDonald Date: Mon, 12 Jun 2023 09:33:47 -0400 Subject: [PATCH 03/13] Updating timing for special cases. --- src/stcal/ramp_fitting/ols_fit.py | 47 +++++++++++++++++++++++++------ src/stcal/ramp_fitting/utils.py | 5 +++- tests/test_ramp_fitting.py | 12 ++++---- 3 files changed, 48 insertions(+), 16 deletions(-) diff --git a/src/stcal/ramp_fitting/ols_fit.py b/src/stcal/ramp_fitting/ols_fit.py index caa02174..e45a87d2 100644 --- a/src/stcal/ramp_fitting/ols_fit.py +++ b/src/stcal/ramp_fitting/ols_fit.py @@ -15,7 +15,9 @@ # HELP!! import sys sys.path.insert(1, "/Users/kmacdonald/code/common") -from general_funcs import dbg_print +from general_funcs import dbg_print, \ + print_ramp_pix, \ + print_ramp_pix_dq ################## DEBUG ################## log = logging.getLogger(__name__) @@ -986,6 +988,8 @@ def ramp_fit_slopes(ramp_data, gain_2d, readnoise_2d, save_opt, weighting): del ff_sect del gdq_sect + # END LOOP + if pixeldq_sect is not None: del pixeldq_sect @@ -1405,7 +1409,9 @@ def ramp_fit_overall( # Divide slopes by total (summed over all integrations) effective # integration time to give count rates. - c_rates = slope_dataset2 / ramp_data.group_time + # XXX JP-3242, push this divide to segment computations + # c_rates = slope_dataset2 / ramp_data.group_time + c_rates = slope_dataset2 # Compress all integration's dq arrays to create 2D PIXELDDQ array for # primary output @@ -2778,14 +2784,16 @@ def fit_lines(data, mask_2d, rn_sect, gain_sect, ngroups, weighting, gdq_sect_r, # Calculate slopes etc. for datasets having either 1 or 2 groups per # integration, and return if ngroups == 1: # process all pixels in 1 group/integration dataset + # XXX JP-3242 slope_s, intercept_s, variance_s, sig_intercept_s, sig_slope_s = \ fit_1_group(slope_s, intercept_s, variance_s, sig_intercept_s, - sig_slope_s, npix, data, c_mask_2d) + sig_slope_s, npix, data, c_mask_2d, ramp_data) return slope_s, intercept_s, variance_s, sig_intercept_s, sig_slope_s if ngroups == 2: # process all pixels in 2 group/integration dataset rn_sect_1d = rn_sect.reshape(npix) + # XXX JP-3242 slope_s, intercept_s, variance_s, sig_intercept_s, sig_slope_s = fit_2_group( slope_s, intercept_s, variance_s, sig_intercept_s, sig_slope_s, npix, data, c_mask_2d, rn_sect_1d, gdq_sect_r, ramp_data) @@ -2802,9 +2810,10 @@ def fit_lines(data, mask_2d, rn_sect, gain_sect, ngroups, weighting, gdq_sect_r, wh_pix_1r = np.where(c_mask_2d[0, :] & (np.logical_not(c_mask_2d[1, :]))) if len(wh_pix_1r[0]) > 0: + # XXX JP-3242 slope_s, intercept_s, variance_s, sig_intercept_s, sig_slope_s = \ fit_single_read(slope_s, intercept_s, variance_s, sig_intercept_s, - sig_slope_s, npix, data, wh_pix_1r) + sig_slope_s, npix, data, wh_pix_1r, ramp_data) del wh_pix_1r @@ -2812,9 +2821,10 @@ def fit_lines(data, mask_2d, rn_sect, gain_sect, ngroups, weighting, gdq_sect_r, # the 0th and 1st group are good, set slope, etc wh_pix_2r = np.where(c_mask_2d.sum(axis=0) == 2) # ramps with 2 good groups + # XXX JP-3242 slope_s, intercept_s, variance_s, sig_slope_s, sig_intercept_s = \ fit_double_read(c_mask_2d, wh_pix_2r, data_masked, slope_s, intercept_s, - variance_s, sig_slope_s, sig_intercept_s, rn_sect) + variance_s, sig_slope_s, sig_intercept_s, rn_sect, ramp_data) del wh_pix_2r @@ -2837,9 +2847,12 @@ def fit_lines(data, mask_2d, rn_sect, gain_sect, ngroups, weighting, gdq_sect_r, sumx, sumxx, sumxy, sumy, nreads_wtd, xvalues = calc_opt_sums( ramp_data, rn_sect, gain_sect, data_masked, c_mask_2d, xvalues, good_pix) + # XXX JP-3242 slope, intercept, sig_slope, sig_intercept = \ calc_opt_fit(nreads_wtd, sumxx, sumx, sumxy, sumy) + slope = slope / ramp_data.group_time + variance = sig_slope**2. # variance due to fit values elif weighting.lower() == 'unweighted': # fit using unweighted weighting @@ -2873,7 +2886,7 @@ def fit_lines(data, mask_2d, rn_sect, gain_sect, ngroups, weighting, gdq_sect_r, def fit_single_read(slope_s, intercept_s, variance_s, sig_intercept_s, - sig_slope_s, npix, data, wh_pix_1r): + sig_slope_s, npix, data, wh_pix_1r, ramp_data): """ For datasets having >2 groups/integrations, for any semiramp in which the 0th group is good and the 1st group is either SAT or CR, set slope, etc. @@ -2904,6 +2917,9 @@ def fit_single_read(slope_s, intercept_s, variance_s, sig_intercept_s, wh_pix_1r : tuple locations of pixels whose only good group is the 0th group + ramp_data : RampData + The ramp data needed for processing, specifically flag values. + Returns ------- slope_s : ndarray @@ -2922,6 +2938,8 @@ def fit_single_read(slope_s, intercept_s, variance_s, sig_intercept_s, 1-D sigma of y-intercepts from fit for data section """ data0_slice = data[0, :, :].reshape(npix) + + # XXX JP-3242 - divide by timing slope_s[wh_pix_1r] = data0_slice[wh_pix_1r] # The following arrays will have values correctly calculated later; for @@ -2935,7 +2953,7 @@ def fit_single_read(slope_s, intercept_s, variance_s, sig_intercept_s, def fit_double_read(mask_2d, wh_pix_2r, data_masked, slope_s, intercept_s, - variance_s, sig_slope_s, sig_intercept_s, rn_sect): + variance_s, sig_slope_s, sig_intercept_s, rn_sect, ramp_data): """ Process all semi-ramps having exactly 2 good groups. May need to optimize later to remove loop over pixels. @@ -2969,6 +2987,9 @@ def fit_double_read(mask_2d, wh_pix_2r, data_masked, slope_s, intercept_s, rn_sect : ndarray 2-D read noise values for all pixels in data section + ramp_data : RampData + The ramp data needed for processing, specifically flag values. + Returns ------- slope_s : ndarray @@ -2999,7 +3020,8 @@ def fit_double_read(mask_2d, wh_pix_2r, data_masked, slope_s, intercept_s, data_semi = data_ramp[mask_2d[:, pixel_ff]] # picks only the 2 diff_data = data_semi[1] - data_semi[0] - slope_s[pixel_ff] = diff_data + # XXX JP-3242 - divide by timing + slope_s[pixel_ff] = diff_data / ramp_data.group_time intercept_s[pixel_ff] = \ data_semi[1] * (1. - second_read) + data_semi[0] * second_read # by geometry variance_s[pixel_ff] = 2.0 * rn * rn @@ -3126,7 +3148,7 @@ def calc_opt_fit(nreads_wtd, sumxx, sumx, sumxy, sumy): def fit_1_group(slope_s, intercept_s, variance_s, sig_intercept_s, - sig_slope_s, npix, data, mask_2d): + sig_slope_s, npix, data, mask_2d, ramp_data): """ This function sets the fitting arrays for datasets having only 1 group per integration. @@ -3157,6 +3179,9 @@ def fit_1_group(slope_s, intercept_s, variance_s, sig_intercept_s, mask_2d : ndarray delineates which channels to fit for each pixel, 2-D bool + ramp_data : RampData + The ramp data needed for processing, specifically flag values. + Returns ------- slope_s : ndarray @@ -3179,6 +3204,7 @@ def fit_1_group(slope_s, intercept_s, variance_s, sig_intercept_s, # time to give the count rate. Recalculate other fit quantities to be # benign. slope_s = data[0, :, :].reshape(npix) + slope_s = slope_s / ramp_data.group_time # The following arrays will have values correctly calculated later; for # now they are just place-holders @@ -3404,6 +3430,9 @@ def fit_2_group(slope_s, intercept_s, variance_s, sig_intercept_s, sig_slope_s, variance_s[one_group_locs] = 1. del one_group_locs + # XXX JP-3242 + slope_s = slope_s / ramp_data.group_time + return slope_s, intercept_s, variance_s, sig_intercept_s, sig_slope_s diff --git a/src/stcal/ramp_fitting/utils.py b/src/stcal/ramp_fitting/utils.py index 8b00daa5..c6b6a568 100644 --- a/src/stcal/ramp_fitting/utils.py +++ b/src/stcal/ramp_fitting/utils.py @@ -292,6 +292,7 @@ def output_optional(self, group_time): self.weights[1. / self.weights > LARGE_VARIANCE_THRESHOLD] = 0. warnings.resetwarnings() + # XXX JP-3242, push this divide to segment computations self.slope_seg /= group_time opt_info = (self.slope_seg, self.sigslope_seg, self.var_p_seg, @@ -736,7 +737,9 @@ def output_integ(ramp_data, slope_int, dq_int, var_p3, var_r3, var_both3): var_r3[var_r3 > LARGE_VARIANCE_THRESHOLD] = 0. var_both3[var_both3 > LARGE_VARIANCE_THRESHOLD] = 0. - data = slope_int / ramp_data.group_time + # XXX JP-3242 - push this computation to the segments + # data = slope_int / ramp_data.group_time + data = slope_int invalid_data = ramp_data.flags_saturated | ramp_data.flags_do_not_use data[np.bitwise_and(dq_int, invalid_data).astype(bool)] = np.nan diff --git a/tests/test_ramp_fitting.py b/tests/test_ramp_fitting.py index 853d33f5..c9a45849 100644 --- a/tests/test_ramp_fitting.py +++ b/tests/test_ramp_fitting.py @@ -8,7 +8,8 @@ # HELP!! import sys sys.path.insert(1, "/Users/kmacdonald/code/common") -from general_funcs import dbg_print +from general_funcs import dbg_print, \ + print_ramp_pix ################## DEBUG ################## @@ -1430,6 +1431,10 @@ def print_real_check(real, check): print("=" * 80) +def print_arr_str(arr): + return np.array2string(arr, max_line_width=np.nan, separator=", ") + + def base_print(label, arr): arr_str = np.array2string(arr, max_line_width=np.nan, separator=", ") print(label) @@ -1580,7 +1585,6 @@ def print_all_info(slopes, cube, optional): print_integ(cube) print_optional(optional) - def print_ramp_data(ramp_data): print(DELIM) print_ramp_data_data(ramp_data) @@ -1589,9 +1593,5 @@ def print_ramp_data(ramp_data): print(DELIM) -def print_ramp_data_data(ramp_data): - base_print("RampData Data:", ramp_data.data) - - def print_ramp_data_dq(ramp_data): base_print("RampData Data Quality:", ramp_data.groupdq) From fea0d2009ab9aa3e3fed8b3117a0e7e375de564a Mon Sep 17 00:00:00 2001 From: Ken MacDonald Date: Wed, 14 Jun 2023 08:22:04 -0400 Subject: [PATCH 04/13] Updating computation of slopes for ramps that have the only good group in the 0th group. --- src/stcal/ramp_fitting/ols_fit.py | 9 ++++++++- tests/test_ramp_fitting.py | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/stcal/ramp_fitting/ols_fit.py b/src/stcal/ramp_fitting/ols_fit.py index e45a87d2..c255b80a 100644 --- a/src/stcal/ramp_fitting/ols_fit.py +++ b/src/stcal/ramp_fitting/ols_fit.py @@ -661,6 +661,7 @@ def ols_ramp_fit_single( """ tstart = time.time() + # import ipdb; ipdb.set_trace() if not ramp_data.suppress_one_group_ramps: # This must be done before the ZEROFRAME replacements to prevent # ZEROFRAME replacement being confused for one good group ramps @@ -2937,10 +2938,16 @@ def fit_single_read(slope_s, intercept_s, variance_s, sig_intercept_s, sig_intercept_s : ndarray 1-D sigma of y-intercepts from fit for data section """ + # XXX current dev data0_slice = data[0, :, :].reshape(npix) # XXX JP-3242 - divide by timing - slope_s[wh_pix_1r] = data0_slice[wh_pix_1r] + # This is ONLY for good 0th group, so need to only distinguish between + # ZEROFRAME and non-ZEROFRAME. + if ramp_data.one_groups_time is not None: + slope_s[wh_pix_1r] = data0_slice[wh_pix_1r] / ramp_data.one_groups_time + else: + slope_s[wh_pix_1r] = data0_slice[wh_pix_1r] / ramp_data.group_time # The following arrays will have values correctly calculated later; for # now they are just place-holders diff --git a/tests/test_ramp_fitting.py b/tests/test_ramp_fitting.py index c9a45849..c5e01e7b 100644 --- a/tests/test_ramp_fitting.py +++ b/tests/test_ramp_fitting.py @@ -940,7 +940,7 @@ def test_only_good_0th_group(): # Because the number of groups used in the first two ramps are different # the variances are expected to be different, even though the slopes # should be the same. - check = np.array([[37.257824, 37.257824, 149.0313]]) + check = np.array([[37.257824, 37.257824, 496.77103]]) np.testing.assert_allclose(sdata, check, tol, tol) check = np.array([[GOOD, GOOD, GOOD]]) From 9dadd211607f3fb95d92e1500cca53e390bcd863 Mon Sep 17 00:00:00 2001 From: Ken MacDonald Date: Wed, 14 Jun 2023 09:16:18 -0400 Subject: [PATCH 05/13] Updating the ZEROFRAME slope computation. --- src/stcal/ramp_fitting/ols_fit.py | 15 ++++++++++++++- src/stcal/ramp_fitting/ramp_fit_class.py | 1 + src/stcal/ramp_fitting/utils.py | 4 +++- tests/test_ramp_fitting.py | 4 ++-- 4 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/stcal/ramp_fitting/ols_fit.py b/src/stcal/ramp_fitting/ols_fit.py index c255b80a..9f05ca33 100644 --- a/src/stcal/ramp_fitting/ols_fit.py +++ b/src/stcal/ramp_fitting/ols_fit.py @@ -670,7 +670,8 @@ def ols_ramp_fit_single( find_0th_one_good_group(ramp_data) if ramp_data.zeroframe is not None: - zframe_locs, cnt = utils.use_zeroframe_for_saturated_ramps(ramp_data) + zframe_mat, zframe_locs, cnt = utils.use_zeroframe_for_saturated_ramps(ramp_data) + ramp_data.zframe_mat = zframe_mat ramp_data.zframe_locs = zframe_locs ramp_data.cnt = cnt @@ -2946,8 +2947,20 @@ def fit_single_read(slope_s, intercept_s, variance_s, sig_intercept_s, # ZEROFRAME and non-ZEROFRAME. if ramp_data.one_groups_time is not None: slope_s[wh_pix_1r] = data0_slice[wh_pix_1r] / ramp_data.one_groups_time + timing = ramp_data.one_groups_time else: slope_s[wh_pix_1r] = data0_slice[wh_pix_1r] / ramp_data.group_time + timing = ramp_data.group_time + + # Adjust slope if ZEROFRAME used. The slope numerator should be + # the frame time if the ZEROFRAME is used. + if ramp_data.zframe_mat is not None: + adjustment = timing / ramp_data.frame_time + good_0th_mat = np.zeros((data0_slice.shape), dtype=np.uint8) + good_0th_mat[wh_pix_1r] = 1 + zframe = ramp_data.zframe_mat[ramp_data.current_integ, :, :].reshape(npix) + adj_mat = good_0th_mat & zframe + slope_s[adj_mat == 1] *= adjustment # The following arrays will have values correctly calculated later; for # now they are just place-holders diff --git a/src/stcal/ramp_fitting/ramp_fit_class.py b/src/stcal/ramp_fitting/ramp_fit_class.py index 084622b1..1eafe1d0 100644 --- a/src/stcal/ramp_fitting/ramp_fit_class.py +++ b/src/stcal/ramp_fitting/ramp_fit_class.py @@ -26,6 +26,7 @@ def __init__(self): self.flags_unreliable_slope = None # ZEROFRAME + self.zframe_mat = None self.zframe_locs = None self.zframe_cnt = 0 self.zeroframe = None diff --git a/src/stcal/ramp_fitting/utils.py b/src/stcal/ramp_fitting/utils.py index c6b6a568..a0257b13 100644 --- a/src/stcal/ramp_fitting/utils.py +++ b/src/stcal/ramp_fitting/utils.py @@ -1601,6 +1601,7 @@ def use_zeroframe_for_saturated_ramps(ramp_data): zframe_locs = [None] * nints cnt = 0 + zframe_mat = np.zeros((nints, nrows, ncols), dtype=np.uint8) for integ in range(nints): intdq = dq[integ, :, :, :] @@ -1623,11 +1624,12 @@ def use_zeroframe_for_saturated_ramps(ramp_data): col_list.append(col) ramp_data.data[integ, 0, row, col] = ramp_data.zeroframe[integ, row, col] ramp_data.groupdq[integ, 0, row, col] = good_flag + zframe_mat[integ, row, col] = 1 cnt = cnt + 1 zframe_locs[integ] = (np.array(row_list, dtype=int), np.array(col_list, dtype=int)) - return zframe_locs, cnt + return zframe_mat, zframe_locs, cnt def groups_saturated_in_integration(intdq, sat_flag, num_sat_groups): diff --git a/tests/test_ramp_fitting.py b/tests/test_ramp_fitting.py index c5e01e7b..3ecc3d58 100644 --- a/tests/test_ramp_fitting.py +++ b/tests/test_ramp_fitting.py @@ -815,7 +815,7 @@ def test_zeroframe(): # Check slopes information sdata, sdq, svp, svr, serr = slopes - check = np.array([[32.78594, 18.62891, 23.787909]]) + check = np.array([[48.965397, 18.628912, 47.863224]]) np.testing.assert_allclose(sdata, check, tol, tol) check = np.array([[GOOD, GOOD, GOOD]]) @@ -836,7 +836,7 @@ def test_zeroframe(): # The third pixel in integration zero has good data # because the zeroframe has good data, so the ramp # is not fully saturated. - check = np.array([[[149.0313, np.nan, 130.40239]], + check = np.array([[[298.0626, np.nan, 652.01196]], [[18.62891, 18.62891, 18.62891]]]) np.testing.assert_allclose(cdata, check, tol, tol) From b53b91b498691b33000340712f4b1311a6b7a2b1 Mon Sep 17 00:00:00 2001 From: Ken MacDonald Date: Wed, 14 Jun 2023 12:10:38 -0400 Subject: [PATCH 06/13] Removing unneeded comments. --- tests/test_ramp_fitting.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/tests/test_ramp_fitting.py b/tests/test_ramp_fitting.py index 3ecc3d58..55e301ed 100644 --- a/tests/test_ramp_fitting.py +++ b/tests/test_ramp_fitting.py @@ -4,15 +4,6 @@ from stcal.ramp_fitting.ramp_fit_class import RampData -################## DEBUG ################## -# HELP!! -import sys -sys.path.insert(1, "/Users/kmacdonald/code/common") -from general_funcs import dbg_print, \ - print_ramp_pix -################## DEBUG ################## - - DELIM = "-" * 70 # single group intergrations fail in the GLS fitting @@ -479,19 +470,12 @@ def run_one_group_ramp_suppression(nints, suppress): In the second integration all pixels have all good groups. """ - # XXX - # This is a bad way to do things. The inputs are bad. The - # group_time is defined independently from nframes, frame_time, - # and groupgap, which is bad. - # Define the data. ngroups, nrows, ncols = 5, 1, 3 dims = (nints, ngroups, nrows, ncols) rnoise, gain = 10, 1 - # nframes, group_time, frame_time = 1, 5.0, 1 nframes, frame_time, groupgap = 1, 1, 0 var = rnoise, gain - # group_time = (nframes + groupgap) * frame_time group_time = 5.0 tm = nframes, group_time, frame_time From a759c59c5d4119794f565042e23d8d220afca9be Mon Sep 17 00:00:00 2001 From: Ken MacDonald Date: Wed, 14 Jun 2023 12:19:58 -0400 Subject: [PATCH 07/13] Removing development code and comments. --- src/stcal/ramp_fitting/ols_fit.py | 23 ++--------------------- src/stcal/ramp_fitting/utils.py | 8 -------- 2 files changed, 2 insertions(+), 29 deletions(-) diff --git a/src/stcal/ramp_fitting/ols_fit.py b/src/stcal/ramp_fitting/ols_fit.py index 9f05ca33..9461e6e0 100644 --- a/src/stcal/ramp_fitting/ols_fit.py +++ b/src/stcal/ramp_fitting/ols_fit.py @@ -11,15 +11,6 @@ from . import utils -################## DEBUG ################## -# HELP!! -import sys -sys.path.insert(1, "/Users/kmacdonald/code/common") -from general_funcs import dbg_print, \ - print_ramp_pix, \ - print_ramp_pix_dq -################## DEBUG ################## - log = logging.getLogger(__name__) log.setLevel(logging.DEBUG) @@ -661,7 +652,6 @@ def ols_ramp_fit_single( """ tstart = time.time() - # import ipdb; ipdb.set_trace() if not ramp_data.suppress_one_group_ramps: # This must be done before the ZEROFRAME replacements to prevent # ZEROFRAME replacement being confused for one good group ramps @@ -1409,10 +1399,8 @@ def ramp_fit_overall( del var_r3 del var_both3 - # Divide slopes by total (summed over all integrations) effective - # integration time to give count rates. - # XXX JP-3242, push this divide to segment computations - # c_rates = slope_dataset2 / ramp_data.group_time + # The slopes per pixel are now computed, except for unusable data + # due to flagging, which is done below. c_rates = slope_dataset2 # Compress all integration's dq arrays to create 2D PIXELDDQ array for @@ -2786,7 +2774,6 @@ def fit_lines(data, mask_2d, rn_sect, gain_sect, ngroups, weighting, gdq_sect_r, # Calculate slopes etc. for datasets having either 1 or 2 groups per # integration, and return if ngroups == 1: # process all pixels in 1 group/integration dataset - # XXX JP-3242 slope_s, intercept_s, variance_s, sig_intercept_s, sig_slope_s = \ fit_1_group(slope_s, intercept_s, variance_s, sig_intercept_s, sig_slope_s, npix, data, c_mask_2d, ramp_data) @@ -2795,7 +2782,6 @@ def fit_lines(data, mask_2d, rn_sect, gain_sect, ngroups, weighting, gdq_sect_r, if ngroups == 2: # process all pixels in 2 group/integration dataset rn_sect_1d = rn_sect.reshape(npix) - # XXX JP-3242 slope_s, intercept_s, variance_s, sig_intercept_s, sig_slope_s = fit_2_group( slope_s, intercept_s, variance_s, sig_intercept_s, sig_slope_s, npix, data, c_mask_2d, rn_sect_1d, gdq_sect_r, ramp_data) @@ -2812,7 +2798,6 @@ def fit_lines(data, mask_2d, rn_sect, gain_sect, ngroups, weighting, gdq_sect_r, wh_pix_1r = np.where(c_mask_2d[0, :] & (np.logical_not(c_mask_2d[1, :]))) if len(wh_pix_1r[0]) > 0: - # XXX JP-3242 slope_s, intercept_s, variance_s, sig_intercept_s, sig_slope_s = \ fit_single_read(slope_s, intercept_s, variance_s, sig_intercept_s, sig_slope_s, npix, data, wh_pix_1r, ramp_data) @@ -2823,7 +2808,6 @@ def fit_lines(data, mask_2d, rn_sect, gain_sect, ngroups, weighting, gdq_sect_r, # the 0th and 1st group are good, set slope, etc wh_pix_2r = np.where(c_mask_2d.sum(axis=0) == 2) # ramps with 2 good groups - # XXX JP-3242 slope_s, intercept_s, variance_s, sig_slope_s, sig_intercept_s = \ fit_double_read(c_mask_2d, wh_pix_2r, data_masked, slope_s, intercept_s, variance_s, sig_slope_s, sig_intercept_s, rn_sect, ramp_data) @@ -2849,7 +2833,6 @@ def fit_lines(data, mask_2d, rn_sect, gain_sect, ngroups, weighting, gdq_sect_r, sumx, sumxx, sumxy, sumy, nreads_wtd, xvalues = calc_opt_sums( ramp_data, rn_sect, gain_sect, data_masked, c_mask_2d, xvalues, good_pix) - # XXX JP-3242 slope, intercept, sig_slope, sig_intercept = \ calc_opt_fit(nreads_wtd, sumxx, sumx, sumxy, sumy) @@ -3040,7 +3023,6 @@ def fit_double_read(mask_2d, wh_pix_2r, data_masked, slope_s, intercept_s, data_semi = data_ramp[mask_2d[:, pixel_ff]] # picks only the 2 diff_data = data_semi[1] - data_semi[0] - # XXX JP-3242 - divide by timing slope_s[pixel_ff] = diff_data / ramp_data.group_time intercept_s[pixel_ff] = \ data_semi[1] * (1. - second_read) + data_semi[0] * second_read # by geometry @@ -3450,7 +3432,6 @@ def fit_2_group(slope_s, intercept_s, variance_s, sig_intercept_s, sig_slope_s, variance_s[one_group_locs] = 1. del one_group_locs - # XXX JP-3242 slope_s = slope_s / ramp_data.group_time return slope_s, intercept_s, variance_s, sig_intercept_s, sig_slope_s diff --git a/src/stcal/ramp_fitting/utils.py b/src/stcal/ramp_fitting/utils.py index a0257b13..24d6a45f 100644 --- a/src/stcal/ramp_fitting/utils.py +++ b/src/stcal/ramp_fitting/utils.py @@ -7,14 +7,6 @@ import warnings -################## DEBUG ################## -# HELP!! -import sys -sys.path.insert(1, "/Users/kmacdonald/code/common") -from general_funcs import dbg_print -################## DEBUG ################## - - log = logging.getLogger(__name__) log.setLevel(logging.DEBUG) From e7e79dcced12dfca142879d021548af127b8f8c7 Mon Sep 17 00:00:00 2001 From: Ken MacDonald Date: Wed, 14 Jun 2023 12:23:25 -0400 Subject: [PATCH 08/13] Cleaning up code. --- src/stcal/ramp_fitting/ols_fit.py | 5 +---- src/stcal/ramp_fitting/utils.py | 2 -- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/src/stcal/ramp_fitting/ols_fit.py b/src/stcal/ramp_fitting/ols_fit.py index 9461e6e0..ce77b633 100644 --- a/src/stcal/ramp_fitting/ols_fit.py +++ b/src/stcal/ramp_fitting/ols_fit.py @@ -2922,12 +2922,9 @@ def fit_single_read(slope_s, intercept_s, variance_s, sig_intercept_s, sig_intercept_s : ndarray 1-D sigma of y-intercepts from fit for data section """ - # XXX current dev data0_slice = data[0, :, :].reshape(npix) - # XXX JP-3242 - divide by timing - # This is ONLY for good 0th group, so need to only distinguish between - # ZEROFRAME and non-ZEROFRAME. + # If the one_groups_time is defined, use it. if ramp_data.one_groups_time is not None: slope_s[wh_pix_1r] = data0_slice[wh_pix_1r] / ramp_data.one_groups_time timing = ramp_data.one_groups_time diff --git a/src/stcal/ramp_fitting/utils.py b/src/stcal/ramp_fitting/utils.py index 24d6a45f..29883295 100644 --- a/src/stcal/ramp_fitting/utils.py +++ b/src/stcal/ramp_fitting/utils.py @@ -729,8 +729,6 @@ def output_integ(ramp_data, slope_int, dq_int, var_p3, var_r3, var_both3): var_r3[var_r3 > LARGE_VARIANCE_THRESHOLD] = 0. var_both3[var_both3 > LARGE_VARIANCE_THRESHOLD] = 0. - # XXX JP-3242 - push this computation to the segments - # data = slope_int / ramp_data.group_time data = slope_int invalid_data = ramp_data.flags_saturated | ramp_data.flags_do_not_use data[np.bitwise_and(dq_int, invalid_data).astype(bool)] = np.nan From c6bd9fa637713c2016789b3931b00dd6377c446f Mon Sep 17 00:00:00 2001 From: Ken MacDonald Date: Wed, 14 Jun 2023 15:03:20 -0400 Subject: [PATCH 09/13] Updating the comments for the computation of one good 0th group ramps. This now is keyed off nframes, rather than groupgap. --- src/stcal/ramp_fitting/ols_fit.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/stcal/ramp_fitting/ols_fit.py b/src/stcal/ramp_fitting/ols_fit.py index ce77b633..1f92a2da 100644 --- a/src/stcal/ramp_fitting/ols_fit.py +++ b/src/stcal/ramp_fitting/ols_fit.py @@ -607,7 +607,12 @@ def find_0th_one_good_group(ramp_data): del bad_1_ ramp_data.one_groups_locs = one_group - # (NFrames + 1) * TFrame / 2 + + # Refer to JP-3242 for derivation. + # Updated: One Group Time = [(TFrame * NFrames * (NFrames + 1)] / [2 * NFrames] + # There is an NFrames term in the numerator and denominator, so when + # cancelled we get: + # One Group Time = (NFrames + 1) * TFrame / 2 ramp_data.one_groups_time = (ramp_data.nframes + 1) * ramp_data.frame_time / 2 @@ -656,7 +661,7 @@ def ols_ramp_fit_single( # This must be done before the ZEROFRAME replacements to prevent # ZEROFRAME replacement being confused for one good group ramps # in the 0th group. - if ramp_data.groupgap > 0: + if ramp_data.nframes > 1: find_0th_one_good_group(ramp_data) if ramp_data.zeroframe is not None: From fe767dca8c25052379f7ea55ae9dc0f6e1327b8c Mon Sep 17 00:00:00 2001 From: Ken MacDonald Date: Fri, 16 Jun 2023 08:08:24 -0400 Subject: [PATCH 10/13] Updating code style failures and removing unnecessary debugging functions. --- src/stcal/ramp_fitting/ols_fit.py | 8 +++----- tests/test_ramp_fitting.py | 13 +------------ 2 files changed, 4 insertions(+), 17 deletions(-) diff --git a/src/stcal/ramp_fitting/ols_fit.py b/src/stcal/ramp_fitting/ols_fit.py index 1f92a2da..1f46c3cd 100644 --- a/src/stcal/ramp_fitting/ols_fit.py +++ b/src/stcal/ramp_fitting/ols_fit.py @@ -870,8 +870,6 @@ def ramp_fit_slopes(ramp_data, gain_2d, readnoise_2d, save_opt, weighting): # Get instrument and exposure data frame_time = ramp_data.frame_time - groupgap = ramp_data.groupgap - nframes = ramp_data.nframes # Get needed sizes and shapes n_int, ngroups, nrows, ncols = data.shape @@ -2931,8 +2929,8 @@ def fit_single_read(slope_s, intercept_s, variance_s, sig_intercept_s, # If the one_groups_time is defined, use it. if ramp_data.one_groups_time is not None: - slope_s[wh_pix_1r] = data0_slice[wh_pix_1r] / ramp_data.one_groups_time - timing = ramp_data.one_groups_time + slope_s[wh_pix_1r] = data0_slice[wh_pix_1r] / ramp_data.one_groups_time + timing = ramp_data.one_groups_time else: slope_s[wh_pix_1r] = data0_slice[wh_pix_1r] / ramp_data.group_time timing = ramp_data.group_time @@ -2944,7 +2942,7 @@ def fit_single_read(slope_s, intercept_s, variance_s, sig_intercept_s, good_0th_mat = np.zeros((data0_slice.shape), dtype=np.uint8) good_0th_mat[wh_pix_1r] = 1 zframe = ramp_data.zframe_mat[ramp_data.current_integ, :, :].reshape(npix) - adj_mat = good_0th_mat & zframe + adj_mat = good_0th_mat & zframe slope_s[adj_mat == 1] *= adjustment # The following arrays will have values correctly calculated later; for diff --git a/tests/test_ramp_fitting.py b/tests/test_ramp_fitting.py index 55e301ed..0fc9ab73 100644 --- a/tests/test_ramp_fitting.py +++ b/tests/test_ramp_fitting.py @@ -474,7 +474,7 @@ def run_one_group_ramp_suppression(nints, suppress): ngroups, nrows, ncols = 5, 1, 3 dims = (nints, ngroups, nrows, ncols) rnoise, gain = 10, 1 - nframes, frame_time, groupgap = 1, 1, 0 + nframes, frame_time = 1, 1 var = rnoise, gain group_time = 5.0 tm = nframes, group_time, frame_time @@ -1568,14 +1568,3 @@ def print_all_info(slopes, cube, optional): print_slopes(slopes) print_integ(cube) print_optional(optional) - -def print_ramp_data(ramp_data): - print(DELIM) - print_ramp_data_data(ramp_data) - print(DELIM) - print_ramp_data_dq(ramp_data) - print(DELIM) - - -def print_ramp_data_dq(ramp_data): - base_print("RampData Data Quality:", ramp_data.groupdq) From 3a30a4cfb553696ac433b6de267254ff3d7cb426 Mon Sep 17 00:00:00 2001 From: Ken MacDonald Date: Wed, 21 Jun 2023 08:52:39 -0400 Subject: [PATCH 11/13] Updating the change log and updating the group time in tests that had bad definitions for group time. --- CHANGES.rst | 7 +++- tests/test_ramp_fitting.py | 84 +++++++++++++++++++------------------- 2 files changed, 48 insertions(+), 43 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index f399ccaf..06dcc3a6 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -4,7 +4,12 @@ Bug Fixes --------- -- +ramp_fitting +~~~~~~~~~~~~ + +- Changing where time division occurs during ramp fitting in order to + properly handle special cases where the time is not group time, such + as when ZEROFRAME data is used, so the time is frame time. [#173] Changes to API -------------- diff --git a/tests/test_ramp_fitting.py b/tests/test_ramp_fitting.py index 0fc9ab73..a50411d4 100644 --- a/tests/test_ramp_fitting.py +++ b/tests/test_ramp_fitting.py @@ -474,9 +474,9 @@ def run_one_group_ramp_suppression(nints, suppress): ngroups, nrows, ncols = 5, 1, 3 dims = (nints, ngroups, nrows, ncols) rnoise, gain = 10, 1 - nframes, frame_time = 1, 1 + nframes, frame_time, groupgap = 1, 1, 0 var = rnoise, gain - group_time = 5.0 + group_time = (nframes + groupgap) * frame_time tm = nframes, group_time, frame_time # Using the above create the classes and arrays. @@ -529,37 +529,37 @@ def test_one_group_ramp_suppressed_one_integration(): # Check slopes information sdata, sdq, svp, svr, serr = slopes - check = np.array([[np.nan, np.nan, 0.20000005]]) + check = np.array([[np.nan, np.nan, 1.0000001]]) np.testing.assert_allclose(sdata, check, tol) check = np.array([[DNU | SAT, DNU, GOOD]]) np.testing.assert_allclose(sdq, check, tol) - check = np.array([[0., 0., 0.01]]) + check = np.array([[0., 0., 0.25]]) np.testing.assert_allclose(svp, check, tol) - check = np.array([[0., 0., 0.19999999]]) + check = np.array([[0., 0., 4.999999]]) np.testing.assert_allclose(svr, check, tol) - check = np.array([[0., 0., 0.45825756]]) + check = np.array([[0., 0., 2.2912877]]) np.testing.assert_allclose(serr, check, tol) # Check slopes information cdata, cdq, cvp, cvr, cerr = cube - check = np.array([[[np.nan, np.nan, 0.20000002]]]) + check = np.array([[[np.nan, np.nan, 1.0000001]]]) np.testing.assert_allclose(cdata, check, tol) check = np.array([[[DNU | SAT, DNU, GOOD]]]) np.testing.assert_allclose(cdq, check, tol) - check = np.array([[[0., 0., 0.01]]]) + check = np.array([[[0., 0., 0.25]]]) np.testing.assert_allclose(cvp, check, tol) - check = np.array([[[0., 0., 0.19999999]]]) + check = np.array([[[0., 0., 4.999999]]]) np.testing.assert_allclose(cvr, check, tol) - check = np.array([[[0., 0., 0.4582576]]]) + check = np.array([[[0., 0., 2.291288]]]) np.testing.assert_allclose(cerr, check, tol) @@ -574,37 +574,37 @@ def test_one_group_ramp_not_suppressed_one_integration(): # Check slopes information sdata, sdq, svp, svr, serr = slopes - check = np.array([[np.nan, 0.2, 0.20000005]]) + check = np.array([[np.nan, 1., 1.0000001]]) np.testing.assert_allclose(sdata, check, tol) check = np.array([[DNU | SAT, GOOD, GOOD]]) np.testing.assert_allclose(sdq, check, tol) - check = np.array([[0., 0.04, 0.01]]) + check = np.array([[0., 1., 0.25]]) np.testing.assert_allclose(svp, check, tol) - check = np.array([[0., 3.9999995, 0.19999999]]) + check = np.array([[0., 100., 5.0000005]]) np.testing.assert_allclose(svr, check, tol) - check = np.array([[0., 2.009975, 0.45825756]]) + check = np.array([[0., 10.049875, 2.291288]]) np.testing.assert_allclose(serr, check, tol) # Check slopes information cdata, cdq, cvp, cvr, cerr = cube - check = np.array([[[np.nan, 0.2, 0.20000002]]]) + check = np.array([[[np.nan, 1., 1.0000001]]]) np.testing.assert_allclose(cdata, check, tol) check = np.array([[[DNU | SAT, GOOD, GOOD]]]) np.testing.assert_allclose(cdq, check, tol) - check = np.array([[[0., 0.04, 0.01]]]) + check = np.array([[[0., 1, 0.25]]]) np.testing.assert_allclose(cvp, check, tol) - check = np.array([[[0., 3.9999995, 0.19999999]]]) + check = np.array([[[0., 100., 5.0000005]]]) np.testing.assert_allclose(cvr, check, tol) - check = np.array([[[0., 2.0099752, 0.4582576]]]) + check = np.array([[[0., 10.049875, 2.291288]]]) np.testing.assert_allclose(cerr, check, tol) @@ -620,42 +620,42 @@ def test_one_group_ramp_suppressed_two_integrations(): # Check slopes information sdata, sdq, svp, svr, serr = slopes - check = np.array([[0.20000002, 0.20000005, 0.20000005]]) + check = np.array([[1.0000001, 1.0000001, 1.0000001]]) np.testing.assert_allclose(sdata, check, tol) check = np.array([[GOOD, GOOD, GOOD]]) np.testing.assert_allclose(sdq, check, tol) - check = np.array([[0.005, 0.01, 0.005]]) + check = np.array([[0.125, 0.25, 0.125]]) np.testing.assert_allclose(svp, check, tol) - check = np.array([[0.19999999, 0.19999999, 0.09999999]]) + check = np.array([[4.999998 , 4.999998 , 2.4999995]]) np.testing.assert_allclose(svr, check, tol) - check = np.array([[0.45276925, 0.45825756, 0.32403702]]) + check = np.array([[2.263846 , 2.2912874, 1.620185]]) np.testing.assert_allclose(serr, check, tol) # Check slopes information cdata, cdq, cvp, cvr, cerr = cube - check = np.array([[[np.nan, np.nan, 0.20000002]], - [[0.20000002, 0.20000002, 0.20000002]]]) + check = np.array([[[np.nan, np.nan, 1.0000001]], + [[1.0000001, 1.0000001, 1.0000001]]]) np.testing.assert_allclose(cdata, check, tol) check = np.array([[[DNU | SAT, DNU, GOOD]], [[GOOD, GOOD, GOOD]]]) np.testing.assert_allclose(cdq, check, tol) - check = np.array([[[0., 0., 0.01]], - [[0.005, 0.01, 0.01]]]) + check = np.array([[[0., 0., 0.25]], + [[0.125, 0.25, 0.25]]]) np.testing.assert_allclose(cvp, check, tol) - check = np.array([[[0., 0., 0.19999999]], - [[0.19999999, 0.19999999, 0.19999999]]]) + check = np.array([[[0., 0., 4.999999]], + [[4.999999, 4.999999, 4.999999]]]) np.testing.assert_allclose(cvr, check, tol) - check = np.array([[[0., 0., 0.4582576]], - [[0.45276922, 0.4582576, 0.4582576]]]) + check = np.array([[[0., 0., 2.291288]], + [[2.2638464, 2.291288, 2.291288]]]) np.testing.assert_allclose(cerr, check, tol) @@ -671,42 +671,42 @@ def test_one_group_ramp_not_suppressed_two_integrations(): # Check slopes information sdata, sdq, svp, svr, serr = slopes - check = np.array([[0.20000002, 0.20000005, 0.20000005]]) + check = np.array([[1.0000001, 1.0000001, 1.0000001]]) np.testing.assert_allclose(sdata, check, tol) check = np.array([[GOOD, GOOD, GOOD]]) np.testing.assert_allclose(sdq, check, tol) - check = np.array([[0.005, 0.008, 0.005]]) + check = np.array([[0.125, 0.2, 0.125]]) np.testing.assert_allclose(svp, check, tol) - check = np.array([[0.19999999, 0.19047618, 0.09999999]]) + check = np.array([[5. , 4.7619047, 2.5000002]]) np.testing.assert_allclose(svr, check, tol) - check = np.array([[0.45276925, 0.44550666, 0.32403702]]) + check = np.array([[2.2638464, 2.2275333, 1.6201853]]) np.testing.assert_allclose(serr, check, tol) # Check slopes information cdata, cdq, cvp, cvr, cerr = cube - check = np.array([[[np.nan, 0.2, 0.20000002]], - [[0.20000002, 0.20000002, 0.20000002]]]) + check = np.array([[[np.nan, 1., 1.0000001]], + [[1.0000001, 1.0000001, 1.0000001]]]) np.testing.assert_allclose(cdata, check, tol) check = np.array([[[DNU | SAT, GOOD, GOOD]], [[GOOD, GOOD, GOOD]]]) np.testing.assert_allclose(cdq, check, tol) - check = np.array([[[0., 0.04, 0.01]], - [[0.005, 0.01, 0.01]]]) + check = np.array([[[0., 1., 0.25]], + [[0.125, 0.25, 0.25]]]) np.testing.assert_allclose(cvp, check, tol) - check = np.array([[[0., 3.9999995, 0.19999999]], - [[0.19999999, 0.19999999, 0.19999999]]]) + check = np.array([[[0., 100., 5.0000005]], + [[5.0000005, 5.0000005, 5.0000005]]]) np.testing.assert_allclose(cvr, check, tol) - check = np.array([[[0., 2.0099752, 0.4582576]], - [[0.45276922, 0.4582576, 0.4582576]]]) + check = np.array([[[0., 10.049875 , 2.291288]], + [[2.2638464, 2.291288 , 2.291288]]]) np.testing.assert_allclose(cerr, check, tol) From 1e70cb659e38a4134ab9e03b0f7e501536eb6398 Mon Sep 17 00:00:00 2001 From: Ken MacDonald Date: Wed, 21 Jun 2023 13:37:18 -0400 Subject: [PATCH 12/13] Corrected optional results calculations. --- src/stcal/ramp_fitting/utils.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/stcal/ramp_fitting/utils.py b/src/stcal/ramp_fitting/utils.py index 29883295..e664a89d 100644 --- a/src/stcal/ramp_fitting/utils.py +++ b/src/stcal/ramp_fitting/utils.py @@ -285,7 +285,7 @@ def output_optional(self, group_time): warnings.resetwarnings() # XXX JP-3242, push this divide to segment computations - self.slope_seg /= group_time + # self.slope_seg /= group_time opt_info = (self.slope_seg, self.sigslope_seg, self.var_p_seg, self.var_r_seg, self.yint_seg, self.sigyint_seg, @@ -632,8 +632,8 @@ def calc_slope_vars(ramp_data, rn_sect, gain_sect, gdq_sect, group_time, max_seg return den_r3, den_p3, num_r3, segs_beg_3 -def calc_pedestal(ramp_data, num_int, slope_int, firstf_int, dq_first, nframes, - groupgap, dropframes1): +def calc_pedestal(ramp_data, num_int, slope_int, firstf_int, dq_first, + nframes, groupgap, dropframes1): """ The pedestal is calculated by extrapolating the final slope for each pixel from its value at the first sample in the integration to an exposure time @@ -672,8 +672,8 @@ def calc_pedestal(ramp_data, num_int, slope_int, firstf_int, dq_first, nframes, pedestal image, 2-D float """ ff_all = firstf_int[num_int, :, :].astype(np.float32) - ped = ff_all - slope_int[num_int, ::] * \ - (((nframes + 1.) / 2. + dropframes1) / (nframes + groupgap)) + tmp = (((nframes + 1.) / 2. + dropframes1) / (nframes + groupgap)) + ped = ff_all - slope_int[num_int, ::] * tmp sat_flag = ramp_data.flags_saturated ped[np.bitwise_and(dq_first, sat_flag) == sat_flag] = 0 From 2f2b9cf70f7f0c3cdc0653ebd5769879f972f5bf Mon Sep 17 00:00:00 2001 From: Ken MacDonald Date: Wed, 21 Jun 2023 15:40:48 -0400 Subject: [PATCH 13/13] Removing unneeded comments. --- src/stcal/ramp_fitting/utils.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/stcal/ramp_fitting/utils.py b/src/stcal/ramp_fitting/utils.py index e664a89d..172e00d2 100644 --- a/src/stcal/ramp_fitting/utils.py +++ b/src/stcal/ramp_fitting/utils.py @@ -284,9 +284,6 @@ def output_optional(self, group_time): self.weights[1. / self.weights > LARGE_VARIANCE_THRESHOLD] = 0. warnings.resetwarnings() - # XXX JP-3242, push this divide to segment computations - # self.slope_seg /= group_time - opt_info = (self.slope_seg, self.sigslope_seg, self.var_p_seg, self.var_r_seg, self.yint_seg, self.sigyint_seg, self.ped_int, self.weights, self.cr_mag_seg)