From e94cdd605c7cbb0acb0b1040d19d5ac3ae269c72 Mon Sep 17 00:00:00 2001 From: Ken MacDonald Date: Mon, 8 Jul 2024 15:26:36 -0400 Subject: [PATCH 1/9] Updating the ramp fit step to accept the likelihood algorithm. Update the detector1 pipeline to handle any problems that can be encountered using the likelihood algorithm. --- jwst/pipeline/calwebb_detector1.py | 18 ++++++++++++++++++ jwst/ramp_fitting/ramp_fit_step.py | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/jwst/pipeline/calwebb_detector1.py b/jwst/pipeline/calwebb_detector1.py index 16087ee73e..dec4d16609 100644 --- a/jwst/pipeline/calwebb_detector1.py +++ b/jwst/pipeline/calwebb_detector1.py @@ -73,6 +73,24 @@ def process(self, input): log.info('Starting calwebb_detector1 ...') + if self.ramp_fit.algorithm.upper() == "LIKELY": + with datamodels.RampModel(input) as ramp: + ngroups = ramp.data.shape[1] + if ngroups < 4: + log_msg = "For the LIKELY ramp fit algorithm NGROUPS must be" + log_msg = f"{log_msg} at least 4. It NGROUPS = {ngroups}." + log_msg = f"{log_msg} Defaulting to OLS_C. Also, the normal" + log_msg = f"{log_msg} pipeline jump detection algorithm will be used." + log.info(log_msg) + self.ramp_fit.algorithm = "OLS_C" + self.jump.skip = False + elif self.jump.skip == False: + log_msg = "For the LIKELY ramp fit algorithm the normal jump detection" + log_msg = f"{log_msg} is skipped, since the LIKELY algorithm uses its" + log_msg = f"{log_msg} jump detection algorithm." + log.info(log_msg) + self.jump.skip = True + # open the input as a RampModel input = datamodels.RampModel(input) diff --git a/jwst/ramp_fitting/ramp_fit_step.py b/jwst/ramp_fitting/ramp_fit_step.py index 1d4291730a..7099dd7b4e 100644 --- a/jwst/ramp_fitting/ramp_fit_step.py +++ b/jwst/ramp_fitting/ramp_fit_step.py @@ -385,7 +385,7 @@ class RampFitStep(Step): class_alias = "ramp_fit" spec = """ - algorithm = option('OLS', 'OLS_C', default='OLS_C') # 'OLS' and 'OLS_C' use the same underlying algorithm, but OLS_C is implemented in C + algorithm = option('OLS', 'OLS_C', 'LIKELY', default='OLS_C') # 'OLS' and 'OLS_C' use the same underlying algorithm, but OLS_C is implemented in C int_name = string(default='') save_opt = boolean(default=False) # Save optional output opt_name = string(default='') From a3be0dd688a2e60f3fc1021a90f57051df273dd7 Mon Sep 17 00:00:00 2001 From: Ken MacDonald Date: Thu, 8 Aug 2024 08:04:27 -0400 Subject: [PATCH 2/9] Updating the change log. --- CHANGES.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 3e5d7a4927..cab31e11a2 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -152,6 +152,9 @@ ramp_fitting - Moved the read noise variance recalculation for CHARGELOSS flagging to the C implementation, when the algorithm is ``OLS_C``. [#8697, spacetelescope/stcal#278] +- Updated the flow of the detector 1 pipeline when selecting the ``LIKELY`` algorithm + for ramp fitting. The ramps must contain a minimum number of groups (4).[#8631] + resample -------- From 84253802991306d1102f04276a9ae6a45bf735f0 Mon Sep 17 00:00:00 2001 From: Ken MacDonald Date: Thu, 5 Sep 2024 08:57:15 -0400 Subject: [PATCH 3/9] The use of the ramp fitting likelihood algorithm should not result in automatically skipping the normal pipeline jump detection algorithm. --- jwst/pipeline/calwebb_detector1.py | 23 ++++++----------------- 1 file changed, 6 insertions(+), 17 deletions(-) diff --git a/jwst/pipeline/calwebb_detector1.py b/jwst/pipeline/calwebb_detector1.py index dec4d16609..7867d12df5 100644 --- a/jwst/pipeline/calwebb_detector1.py +++ b/jwst/pipeline/calwebb_detector1.py @@ -73,26 +73,15 @@ def process(self, input): log.info('Starting calwebb_detector1 ...') + # open the input as a RampModel + input = datamodels.RampModel(input) + if self.ramp_fit.algorithm.upper() == "LIKELY": - with datamodels.RampModel(input) as ramp: - ngroups = ramp.data.shape[1] + ngroups = input.data.shape[1] if ngroups < 4: - log_msg = "For the LIKELY ramp fit algorithm NGROUPS must be" - log_msg = f"{log_msg} at least 4. It NGROUPS = {ngroups}." - log_msg = f"{log_msg} Defaulting to OLS_C. Also, the normal" - log_msg = f"{log_msg} pipeline jump detection algorithm will be used." - log.info(log_msg) + log.warning(f"For the LIKELY ramp fit algorithm NGROUPS must be" + f" at least 4, but NGROUPS = {ngroups}.") self.ramp_fit.algorithm = "OLS_C" - self.jump.skip = False - elif self.jump.skip == False: - log_msg = "For the LIKELY ramp fit algorithm the normal jump detection" - log_msg = f"{log_msg} is skipped, since the LIKELY algorithm uses its" - log_msg = f"{log_msg} jump detection algorithm." - log.info(log_msg) - self.jump.skip = True - - # open the input as a RampModel - input = datamodels.RampModel(input) # propagate output_dir to steps that might need it self.dark_current.output_dir = self.output_dir From bd27d2db667fecbebaa8d356e986464e494ac3dd Mon Sep 17 00:00:00 2001 From: Ken MacDonald Date: Mon, 9 Sep 2024 12:22:32 -0400 Subject: [PATCH 4/9] Making changes due to code review. --- jwst/pipeline/calwebb_detector1.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/jwst/pipeline/calwebb_detector1.py b/jwst/pipeline/calwebb_detector1.py index 7867d12df5..885fa0952e 100644 --- a/jwst/pipeline/calwebb_detector1.py +++ b/jwst/pipeline/calwebb_detector1.py @@ -80,7 +80,8 @@ def process(self, input): ngroups = input.data.shape[1] if ngroups < 4: log.warning(f"For the LIKELY ramp fit algorithm NGROUPS must be" - f" at least 4, but NGROUPS = {ngroups}.") + f" at least 4, but NGROUPS = {ngroups}. Switching" + f" algorithm to OLS_C.") self.ramp_fit.algorithm = "OLS_C" # propagate output_dir to steps that might need it From 7673fc8f950c3610f70b33205b4cd3efea9b138d Mon Sep 17 00:00:00 2001 From: Ken MacDonald Date: Thu, 19 Sep 2024 13:57:28 -0400 Subject: [PATCH 5/9] Removing code from the pipeline that is a duplication of code in STCAL. --- jwst/pipeline/calwebb_detector1.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/jwst/pipeline/calwebb_detector1.py b/jwst/pipeline/calwebb_detector1.py index 885fa0952e..16087ee73e 100644 --- a/jwst/pipeline/calwebb_detector1.py +++ b/jwst/pipeline/calwebb_detector1.py @@ -76,14 +76,6 @@ def process(self, input): # open the input as a RampModel input = datamodels.RampModel(input) - if self.ramp_fit.algorithm.upper() == "LIKELY": - ngroups = input.data.shape[1] - if ngroups < 4: - log.warning(f"For the LIKELY ramp fit algorithm NGROUPS must be" - f" at least 4, but NGROUPS = {ngroups}. Switching" - f" algorithm to OLS_C.") - self.ramp_fit.algorithm = "OLS_C" - # propagate output_dir to steps that might need it self.dark_current.output_dir = self.output_dir self.ramp_fit.output_dir = self.output_dir From 940312a8b146b67ac81086ba95c59303639e215a Mon Sep 17 00:00:00 2001 From: Ken MacDonald Date: Thu, 19 Sep 2024 13:59:09 -0400 Subject: [PATCH 6/9] Adding information about the likelihood algorithm in the ramp fitting documentation. --- docs/jwst/ramp_fitting/description.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/jwst/ramp_fitting/description.rst b/docs/jwst/ramp_fitting/description.rst index d4d63862a6..4d0e5329fc 100644 --- a/docs/jwst/ramp_fitting/description.rst +++ b/docs/jwst/ramp_fitting/description.rst @@ -54,6 +54,11 @@ JWST-specific features are described later within this document. Upon successful completion of this step, the status keyword S_RAMP will be set to "COMPLETE". +There is a new algorithm available for testing in ramp fitting, which is the +likelihood algorithm. It is selected by setting ``--ramp_fitting.algorithm=LIKELY``. +The details are in the ``stcal`` documentation at +:ref:`Likelihood Algorithm Details `. + Multiprocessing --------------- This step has the option of running in multiprocessing mode. In that mode it will From 8773c45bde0082df3ced7b370cfc317e60f8d5de Mon Sep 17 00:00:00 2001 From: Ken MacDonald Date: Thu, 19 Sep 2024 14:09:02 -0400 Subject: [PATCH 7/9] Making sure the minimum number of groups is met when selecting the LIKELY algorithm for ramp fitting. --- jwst/ramp_fitting/ramp_fit_step.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/jwst/ramp_fitting/ramp_fit_step.py b/jwst/ramp_fitting/ramp_fit_step.py index 7099dd7b4e..08acebda1b 100644 --- a/jwst/ramp_fitting/ramp_fit_step.py +++ b/jwst/ramp_fitting/ramp_fit_step.py @@ -5,6 +5,8 @@ from stcal.ramp_fitting import ramp_fit from stcal.ramp_fitting import utils +from stcal.ramp_fitting.likely_fit import LIKELY_MIN_NGROUPS + from stcal.ramp_fitting.utils import LARGE_VARIANCE from stcal.ramp_fitting.utils import LARGE_VARIANCE_THRESHOLD @@ -414,8 +416,16 @@ def process(self, input): readnoise_filename = self.get_reference_file(input_model, 'readnoise') gain_filename = self.get_reference_file(input_model, 'gain') - log.info('Using READNOISE reference file: %s', readnoise_filename) - log.info('Using GAIN reference file: %s', gain_filename) + ngroups = input_model.data.shape[1] + if self.algorithm.upper() == "LIKELY" and ngroups < LIKELY_MIN_NGROUPS: + log.info(f"When selecting the LIKELY ramp fitting algorithm the" + " ngroups needs to be a minimum of {likely_fit.LIKELY_MIN_NGROUPS}," + " but ngroups = {ngroups}. Due to this, the ramp fitting algorithm" + " is being changed to OLS_C") + self.algorithm = "OLS_C" + + log.info(f"Using READNOISE reference file: {readnoise_filename}") + log.info(f"Using GAIN reference file: {gain_filename}") with datamodels.ReadnoiseModel(readnoise_filename) as readnoise_model, \ datamodels.GainModel(gain_filename) as gain_model: @@ -432,8 +442,8 @@ def process(self, input): readnoise_2d, gain_2d = get_reference_file_subarrays( input_model, readnoise_model, gain_model, frames_per_group) - log.info('Using algorithm = %s' % self.algorithm) - log.info('Using weighting = %s' % self.weighting) + log.info(f"Using algorithm = {self.algorithm}") + log.info(f"Using weighting = {self.weighting}") buffsize = ramp_fit.BUFSIZE if self.algorithm == "GLS": From 085442af35c26974d61244c9703afe653cfaa4d6 Mon Sep 17 00:00:00 2001 From: Ken MacDonald Date: Thu, 19 Sep 2024 15:30:07 -0400 Subject: [PATCH 8/9] Updating reference to minimum number of groups defined in another module. --- jwst/ramp_fitting/ramp_fit_step.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jwst/ramp_fitting/ramp_fit_step.py b/jwst/ramp_fitting/ramp_fit_step.py index 08acebda1b..522afce695 100644 --- a/jwst/ramp_fitting/ramp_fit_step.py +++ b/jwst/ramp_fitting/ramp_fit_step.py @@ -419,7 +419,7 @@ def process(self, input): ngroups = input_model.data.shape[1] if self.algorithm.upper() == "LIKELY" and ngroups < LIKELY_MIN_NGROUPS: log.info(f"When selecting the LIKELY ramp fitting algorithm the" - " ngroups needs to be a minimum of {likely_fit.LIKELY_MIN_NGROUPS}," + " ngroups needs to be a minimum of {LIKELY_MIN_NGROUPS}," " but ngroups = {ngroups}. Due to this, the ramp fitting algorithm" " is being changed to OLS_C") self.algorithm = "OLS_C" From c732a74c9cbb4aca614e7b3bdbcd854328c4d565 Mon Sep 17 00:00:00 2001 From: Ken MacDonald Date: Thu, 19 Sep 2024 15:42:19 -0400 Subject: [PATCH 9/9] Updated f-string. --- jwst/ramp_fitting/ramp_fit_step.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/jwst/ramp_fitting/ramp_fit_step.py b/jwst/ramp_fitting/ramp_fit_step.py index 522afce695..12c34b29df 100644 --- a/jwst/ramp_fitting/ramp_fit_step.py +++ b/jwst/ramp_fitting/ramp_fit_step.py @@ -419,9 +419,9 @@ def process(self, input): ngroups = input_model.data.shape[1] if self.algorithm.upper() == "LIKELY" and ngroups < LIKELY_MIN_NGROUPS: log.info(f"When selecting the LIKELY ramp fitting algorithm the" - " ngroups needs to be a minimum of {LIKELY_MIN_NGROUPS}," - " but ngroups = {ngroups}. Due to this, the ramp fitting algorithm" - " is being changed to OLS_C") + f" ngroups needs to be a minimum of {LIKELY_MIN_NGROUPS}," + f" but ngroups = {ngroups}. Due to this, the ramp fitting algorithm" + f" is being changed to OLS_C") self.algorithm = "OLS_C" log.info(f"Using READNOISE reference file: {readnoise_filename}")