Added program data to testing data, updated filename management

oirlab · Jan 8, 2025 · 29d084d · 29d084d
1 parent 270f7e6
commit 29d084d
Show file tree

Hide file tree

Showing 66 changed files with 825 additions and 525 deletions.
diff --git a/.gitattributes b/.gitattributes
@@ -1,6 +1,4 @@
-liger_iris_pipeline/tests/data/2024A-P123-044_IRIS_IMG1_SKY-SIM-Y_LVL1_0001-00.fits filter=lfs diff=lfs merge=lfs -text
-liger_iris_pipeline/tests/data/2024A-P123-044_Liger_IMG1_SCI-J1458+1013-SIM-Y_LVL1_0001.fits filter=lfs diff=lfs merge=lfs -text
-liger_iris_pipeline/tests/data/2024A-P123-044_Liger_IMG_SCI-J1458+1013-SIM-Y_LVL1_0001-00.fits filter=lfs diff=lfs merge=lfs -text
-liger_iris_pipeline/tests/data/2024A-P123-044_Liger_IMG_SCI-J1458+1013-SIM-Y_LVL1_0001.fits filter=lfs diff=lfs merge=lfs -text
-liger_iris_pipeline/tests/data/2024A-P123-044_IRIS_IMG1_SCI-J1458+1013-SIM-Y_LVL0_0001-00.fits filter=lfs diff=lfs merge=lfs -text
-liger_iris_pipeline/tests/data/2024A-P123-044_IRIS_IMG1_SCI-J1458+1013-SIM-Y_LVL1_0001-00.fits filter=lfs diff=lfs merge=lfs -text
+liger_iris_pipeline/tests/data/2024B-P123-008_IRIS_IMG1_SKY-J1458+1013-Y-4.0_LVL1_0001-00.fits filter=lfs diff=lfs merge=lfs -text
+liger_iris_pipeline/tests/data/2024B-P123-008_Liger_IMG_SCI-J1458+1013-Y-10.0_LVL1_0001-00.fits filter=lfs diff=lfs merge=lfs -text
+liger_iris_pipeline/tests/data/2024B-P123-008_IRIS_IMG1_SCI-J1458+1013-Y-4.0_LVL0_0001-00.fits filter=lfs diff=lfs merge=lfs -text
+liger_iris_pipeline/tests/data/2024B-P123-008_IRIS_IMG1_SCI-J1458+1013-Y-4.0_LVL1_0001-00.fits filter=lfs diff=lfs merge=lfs -text
diff --git a/docs/assign_wcs/main.rst b/docs/assign_wcs/main.rst
@@ -28,8 +28,6 @@ See an example script to process a file with FITS WCS keywords in the header::
     import liger_iris_pipeline
     import astropy.units as u
 
-    liger_iris_pipeline.monkeypatch_jwst_datamodels()
-
     input_filename ="iris_sim_gc_filterKN3_fix.fits"
-    output = liger_iris_pipeline.assign_wcs.AssignWcsStep.call(input_filename)
+    output = liger_iris_pipeline.assign_wcs.AssignWCSStep.call(input_filename)
     print(output.meta.wcs([0,4096]*u.pix,[0,4096]*u.pix))
diff --git a/docs/available-steps.rst b/docs/available-steps.rst
@@ -4,10 +4,9 @@ Available algorithms
 .. toctree::
    :maxdepth: 2
 
-   drsrop_pipeline.rst
    dq_init/index.rst
    background/index.rst
-   dark_current/index.rst
+   dark_subtraction/index.rst
    normalize/index.rst
    pipeline/index.rst
    flatfield/index.rst

diff --git a/docs/background/index.rst b/docs/background/index.rst
diff --git a/docs/dark_current/arguments.rst → docs/dark_subtraction/arguments.rst b/docs/dark_current/arguments.rst → docs/dark_subtraction/arguments.rst
diff --git a/docs/dark_current/description.rst → docs/dark_subtraction/description.rst b/docs/dark_current/description.rst → docs/dark_subtraction/description.rst
diff --git a/docs/dark_current/index.rst → docs/dark_subtraction/index.rst b/docs/dark_current/index.rst → docs/dark_subtraction/index.rst
@@ -1,4 +1,4 @@
-.. _dark_current_step:
+.. _dark_step:
 
 ========================
 Dark Current Subtraction
@@ -12,4 +12,4 @@ Dark Current Subtraction
    arguments.rst
 
 
-.. automodapi:: liger_iris_pipeline.dark_current
+.. automodapi:: liger_iris_pipeline.dark_subtraction
diff --git a/docs/dark_current/reference_files.rst → docs/dark_subtraction/reference_files.rst b/docs/dark_current/reference_files.rst → docs/dark_subtraction/reference_files.rst
@@ -6,4 +6,4 @@ The ``dark`` step uses a DARK reference file.
 
 The ``dark`` reference file by default is retrieved from the CRDS,
 to use instead a local file, specify the ``override_dark`` keyword
-in the configuration file specifying a path to a :py:class:`IRISImageModel`.
+in the configuration file specifying a path to a :py:class:`ImagerModel`.
diff --git a/docs/index.rst b/docs/index.rst
@@ -2,7 +2,7 @@
 liger_iris_pipeline Documentation
 *********************************
 
-The Liger-IRIS Data Reduction System is based on the suite of packages from Space Telescope Science Institute
+The Liger-IRIS Data Reduction System (DRS) is based on the suite of packages from Space Telescope Science Institute
 used to process data from JWST and NGRST.
 
 With ``stpipe`` and ``stdatamodels`` we can configure each step of a pipeline through one or more text based .INI style files,

diff --git a/docs/pipeline/preprocess_flatfield.rst → docs/pipeline/create_flatfield.rst b/docs/pipeline/preprocess_flatfield.rst → docs/pipeline/create_flatfield.rst
diff --git a/docs/pipeline/image2.rst → docs/pipeline/imager_stage2.rst b/docs/pipeline/image2.rst → docs/pipeline/imager_stage2.rst
@@ -4,7 +4,7 @@ image2: Stage 2 Imaging Processing
 ==================================
 
 :Config: image2.cfg
-:Class: `~liger_iris_pipeline.pipeline.ProcessImagerL2Pipeline`
+:Class: `~liger_iris_pipeline.pipeline.ImagerStage2Pipeline`
 
 Stage 2 imaging processing applies additional instrumental corrections and
 calibrations that result in a fully calibrated individual exposure. 

diff --git a/docs/pipeline/index.rst b/docs/pipeline/index.rst
@@ -7,7 +7,7 @@ Pipeline Modules
    :maxdepth: 2
 
    main.rst
-   image2.rst
-   preprocess_flatfield.rst
+   imager_stage2.rst
+   create_flatfield.rst
 
 .. automodapi:: liger_iris_pipeline.pipeline
diff --git a/docs/drsrop_pipeline.rst → docs/pipeline/stage1.rst b/docs/drsrop_pipeline.rst → docs/pipeline/stage1.rst
@@ -14,34 +14,13 @@ Non-linearity correction step corrects for the non-linear response of the detect
 Detector Readout Sampling
 -------------------------
 The H4RG detecors are readout in non-destructive reads and sampling algorithms are used to estimate the accumulated electrons in the detector for an integration time. The sampling algorithms currently implemented in the pipeline are
-- Correlated Double Sampling
-- Multi Correlated Double Sampling
+- (Multi)-Correlated Double Sampling
 - Up-the-Ramp Sampling
 
-Requirements
-------------
-The sampling algorithms use the drsrop_clib module.
-https://github.com/oirlab/iris_readout
-
 Running the Examples
 --------------------
-There is a example run in the liger_iris_pipeline/readout/tests directory. The sample ramp is given in the sample_ramp.fits.
-sampling.cfg gives the configurations for the pipeline
-
-``sampling.cfg``:
-
-.. code-block:: ini
-
-    name = "rop"
-    class = "liger_iris_pipeline.pipeline.ROPPipeline"
-    save_results = True
-        [steps]
-          [[nonlincorr]]
-          [[readoutsamp]]
-           mode='mcds'
-
-The sampling mode is set by the ``mode`` keyword which can be ``mcds`` or ``utr``. MCDS algorithm also requires the group number, the number of reads to be co-added. This is currently hardcoded in this version.
-
+There is a example run in the liger_iris_pipeline/readout directory. The sample ramp is given in the sample_ramp.fits.
+sampling.cfg gives the configurations for the pipeline.
 
 Execute the pipeline from the command line
 ------------------------------------------

diff --git a/docs/background/arguments.rst → docs/sky_subtraction/arguments.rst b/docs/background/arguments.rst → docs/sky_subtraction/arguments.rst
diff --git a/docs/background/description.rst → docs/sky_subtraction/description.rst b/docs/background/description.rst → docs/sky_subtraction/description.rst
diff --git a/docs/sky_subtraction/index.rst b/docs/sky_subtraction/index.rst
@@ -0,0 +1,14 @@
+.. _sky_subtraction_step:
+
+====================
+Sky Subtraction Step
+====================
+
+.. toctree::
+   :maxdepth: 2
+
+   description.rst
+   arguments.rst
+   reference_files.rst
+
+.. automodapi:: liger_iris_pipeline.sky_subtraction
diff --git a/docs/background/reference_files.rst → docs/sky_subtraction/reference_files.rst b/docs/background/reference_files.rst → docs/sky_subtraction/reference_files.rst
diff --git a/docs/subarrays.rst b/docs/subarrays.rst
@@ -1,43 +1,42 @@
 *********************
-Support for subarrays
+Support for Subarrays
 *********************
 
 Support for subarrays is currently only implemented for the imager and it supports
 datasets where only a custom subset of the 2D array is observed.
 
-The keywords of :py:class:`IRISImageModel` which defines the parameters of the
-subarray are::
+The keywords of :py:class:`ImagerModel` which defines the parameters of the subarray are::
 
     model.meta.subarray.name = "CUSTOM"
     model.meta.subarray.id = 1
     model.meta.subarray.xstart = xstart + 1
     model.meta.subarray.ystart = ystart + 1
     model.meta.subarray.xsize = xsize
     model.meta.subarray.ysize = ysize
+    model.meta.subarray.detysiz = detysiz
+    model.meta.subarray.detxsiz = detxsiz
+    model.meta.subarray.fastaxis = 0
+    model.meta.subarray.slowaxis = 1
 
-Consider that following the FITS conventions the `xstart` and `ystart` keywords
-are 1-based, therefore the default `xstart` is 1 and if you are slicing an
-array in Python, you should add 1 to the keyword before saving it into the metadata.
-`subarray.id` is saved into the FITS keyword `SUBARRID` and should be 0 for full
-frames, 1 for the first subarray and so on.
+Here the `xstart` and `ystart` keywords are converted from 0-based to 1-based indexing.
+`xstart` and `ystart` are 1 by default. `subarray.id` is saved into the FITS keyword `SUBARRID` and should be 0 for full
+frames, 1 for the first subarray and so on. The name of an entire frame is "FULL", whereas subarray names can be anything.
 
-The name of an entire frame is "FULL".
-
-Subarrays and reference files
+Subarrays and Reference Files
 =============================
 
 Flat frames, darks and background files either in CRDS or using local overrides
-can either be saved as subarrays
-or can be saved as full frames. In case they are saved as full frames, after being
-accessed they are sliced according to the metadata in the input subarray.
+can either be saved as subarrays or can be saved as full frames.
+In case they are saved as full frames, after being accessed they are sliced
+according to the metadata in the input subarray.
 
-Example usage
+Example Usage
 =============
 
-As usage examples, check the notebooks or the ``test_image2.py`` script in the
+As usage examples, check the notebooks or the ``test_imager_stage2.py`` script in the
 `unit tests folder in the repository <https://github.com/oirlab/liger_iris_pipeline/tree/master/liger_iris_pipeline/tests>`_
 
-Related steps
+Related Steps
 =============
 
 .. toctree::

diff --git a/liger_iris_pipeline/assign_wcs/assign_wcs.py b/liger_iris_pipeline/assign_wcs/assign_wcs.py
@@ -64,5 +64,5 @@ def load_wcs(input_model, reference_files={}):
 
         wcs = WCS(pipeline)
         output_model.meta.wcs = wcs
-        output_model.meta.cal_step.assign_wcs = "COMPLETE"
+
     return output_model
diff --git a/liger_iris_pipeline/assign_wcs/assign_wcs_step.py b/liger_iris_pipeline/assign_wcs/assign_wcs_step.py
@@ -20,6 +20,8 @@ class AssignWCSStep(LigerIRISStep):
     # eventually ['distortion' , 'specwcs', 'wavelengthrange']
     reference_file_types = []
 
+    class_alias = "assign_wcs"
+
     def process(self, input, *args, **kwargs):
         reference_file_names = {}
         if isinstance(input, str):
@@ -33,7 +35,7 @@ def process(self, input, *args, **kwargs):
             log.warning("assign_wcs expects ImageModel as input.")
             log.warning("Skipping assign_wcs step.")
             result = input_model.copy()
-            result.meta.cal_step.assign_wcs = "SKIPPED"
+            self.status = "SKIPPED"
         else:
             # Get reference files
             for reftype in self.reference_file_types:
@@ -43,6 +45,7 @@ def process(self, input, *args, **kwargs):
 
             # Assign wcs
             result = load_wcs(input_model, reference_file_names)
+            self.status = "COMPLETE"
 
         # Close model if opened manually
         if isinstance(input, str):

diff --git a/liger_iris_pipeline/associations/__init__.py b/liger_iris_pipeline/associations/__init__.py
@@ -1,14 +1,16 @@
 # https://jwst-pipeline.readthedocs.io/en/latest/jwst/associations/association_reference.html#ref-asn-core-methods
 
 from .association_base import LigerIRISAssociation
-from .imager_level0 import ImagerL0Association
-from .imager_level1 import ImagerL1Association
+from .level0 import L0Association
+from .level1 import L1Association
+from .subarray import SubarrayAssociation
 from .utils import load_asn
 
 __all__ = [
     "LigerIRISAssociation",
-    "ImagerL0Association",
-    "ImagerL1Association",
+    "L0Association",
+    "L1Association",
+    "SubarrayAssociation",
     "load_asn"
 ]
 

diff --git a/liger_iris_pipeline/associations/association_base.py b/liger_iris_pipeline/associations/association_base.py
@@ -56,12 +56,12 @@ def from_product(cls, product : dict):
         return asn
 
     @classmethod
-    def from_member(cls, filename : str):
-        input_model = datamodels.open(filename)
+    def from_member(cls, member : str | datamodels.LigerIRISDataModel):
+        input_model = datamodels.open(member)
         product = {
             "members": [
                 {
-                    "expname": filename,
+                    "expname": input_model.filename,
                     "exptype": input_model.meta.exposure.type,
                 },
             ]

diff --git a/liger_iris_pipeline/associations/imager_level0.py b/liger_iris_pipeline/associations/imager_level0.py
diff --git a/liger_iris_pipeline/associations/level0.py b/liger_iris_pipeline/associations/level0.py
@@ -0,0 +1,6 @@
+from . import LigerIRISAssociation
+
+__all__ = ['L0Association']
+
+class L0Association(LigerIRISAssociation):
+    pass
diff --git a/...is_pipeline/associations/imager_level1.py → liger_iris_pipeline/associations/level1.py b/...is_pipeline/associations/imager_level1.py → liger_iris_pipeline/associations/level1.py
@@ -2,9 +2,9 @@
 
 __all__ = ['ImagerL1Association']
 
-class ImagerL1Association(LigerIRISAssociation):
+class L1Association(LigerIRISAssociation):
     """
-    IRIS Imager Level 1 Association
+    Imager Level 1 Association
     TODO:
         Implement subarrays in the association.
     """

diff --git a/liger_iris_pipeline/associations/subarray.py b/liger_iris_pipeline/associations/subarray.py
@@ -0,0 +1,9 @@
+from . import LigerIRISAssociation
+
+__all__ = ['SubarrayAssociation']
+
+class SubarrayAssociation(LigerIRISAssociation):
+    """
+    Subarray Association for any level.
+    """
+    pass