Merge of ywalk and hvds branches (#237)

* Add implementation of high voltage dependent sensitivity (COSPIP-54)

* Make ywalk depend on ycorr rather than xcorr.  Clean up docstring.
Close walk ref file

* Change correction from divisive to multiplicative
Fix type in filetype for hvdstab

* Add warning if ywalk correction array is not 32x1024

* Add x and y walkcorr switches to the list of switches to leave alone
when doing --only_csum processing with the --raw_csum flag not set

* Add DEADCORR to list of steps to leave unchanged when --csum-only is selected

* Bugfix: left off a comma at the end of a line

calcos/calcos.py

@@ -1269,7 +1269,8 @@ def resetSwitches(self):
         if self.cl_args["raw_csum_coords"]:
             leave_unchanged = []        # reset all switches to OMIT
         else:
-            leave_unchanged = ["tempcorr", "geocorr", "dgeocorr", "igeocorr", "randcorr"]
+            leave_unchanged = ["tempcorr", "geocorr", "dgeocorr", "igeocorr",
+                               "deadcorr", "xwlkcorr", "ywlkcorr", "randcorr"]
 
         for obs in self.obs:
             for key in obs.switches:
@@ -1418,7 +1419,6 @@ def missingRefFiles(self):
         missing = {}            # reference file is not accessible
         wrong_filetype = {}     # wrong FILETYPE
         bad_version = {}        # inconsistent version strings
-
         # temp is a temporary dictionary with just min_ver and filetype,
         # for readability; these and other values will be copied to ref,
         # which is used for the argument to findRefFile.
@@ -1448,7 +1448,8 @@ def missingRefFiles(self):
             "tracetab": ["2.0", "1D SPECTRAL TRACE TABLE"],
             "proftab": ["2.0", "2D SPECTRUM PROFILE TABLE"],
             "twozxtab": ["2.0", "TWO-ZONE SPECTRAL EXTRACTION PARAMETERS TABLE"],
-            "spottab": ["2.0", "TRANSIENT BAD PIXEL REFERENCE TABLE"]
+            "spottab": ["2.0", "TRANSIENT BAD PIXEL REFERENCE TABLE"],
+            "hvdstab": ["3.4", "HV SENSITIVITY TABLE TO CORRECT EPSILON BASED ON EXPOSURE HV"]
         }
         # The contents of these dictionaries must agree with what
         # cosutil.findRefFile expects.
@@ -1469,6 +1470,10 @@ def missingRefFiles(self):
             cosutil.findRefFile(ref["flatfile"],
                                 missing, wrong_filetype, bad_version)
 
+        if switches["hvdscorr"] == "PERFORM":
+            cosutil.findRefFile(ref["hvdstab"],
+                                missing, wrong_filetype, bad_version)
+
         if switches["brstcorr"] == "PERFORM":
             cosutil.findRefFile(ref["brsttab"],
                                 missing, wrong_filetype, bad_version)
@@ -1634,7 +1639,7 @@ def globalSwitches(self):
             self.global_switches["any"] = "PERFORM"
         for key in ["badtcorr", "brstcorr", "deadcorr", "doppcorr",
                     "dqicorr",  "flatcorr", "geocorr",
-                    "dgeocorr", "helcorr",
+                    "dgeocorr", "helcorr", "hvdscorr",
                     "phacorr",  "randcorr", "tempcorr", "x1dcorr",
                     "wavecorr", "trcecorr", "algncorr"]:
             if switches[key] == "PERFORM":

calcos/getinfo.py

@@ -265,7 +265,7 @@ def getSwitchValues(phdr):
                 "dgeocorr", "xwlkcorr", "ywlkcorr", "trcecorr", "algncorr",
                 "deadcorr", "flatcorr", "doppcorr", "helcorr", "phacorr",
                 "brstcorr", "badtcorr", "x1dcorr", "wavecorr", "backcorr",
-                "fluxcorr", "photcorr", "tdscorr", "statflag"]:
+                "fluxcorr", "photcorr", "tdscorr", "hvdscorr", "statflag"]:
         switches[key]  = cosutil.getSwitch(phdr, key)
 
     return switches
@@ -303,7 +303,7 @@ def getRefFileNames(phdr):
                 "phatab", "brsttab", "badttab", "tracetab",
                 "xtractab", "lamptab", "disptab", "fluxtab",
                 "imphttab", "phottab", "spwcstab", "wcptab",
-                "tdstab", "proftab"]:
+                "tdstab", "proftab", "hvdstab"]:
         reffiles[key+"_hdr"] = phdr.get(key, default=NOT_APPLICABLE)
         reffiles[key] = cosutil.expandFileName(reffiles[key+"_hdr"])
 

calcos/timetag.py

@@ -243,6 +243,8 @@ def timetagBasicCalibration(input, inpha, outtag,
 
     doFlatcorr(events, info, switches, reffiles, phdr, headers[1])
 
+    doHvdscorr(events, info, switches, reffiles, phdr, headers[1])
+
     phdr["wavecals"] = ""               # initial value
     if info["tagflash"]:
         cosutil.printSwitch("WAVECORR", switches)
@@ -2111,7 +2113,14 @@ def doYWalkcorr(events, info, switches, reffiles, phdr):
         cosutil.printSwitch("YWLKCORR", switches)
         if switches["ywlkcorr"] == "PERFORM":
             cosutil.printRef("YWLKFILE", reffiles)
-            ycorrection = walkCorrection(events.field('xcorr'),
+            correct_size = (32, 1024)
+            if YWalkReffile_hasWrongSize(reffiles["ywlkfile"], info["segment"], correct_size):
+                cosutil.printWarning("You are running CALCOS with a YWLKFILE that is of different")
+                cosutil.printContinuation("dimensions than expected by the YWLKCORR routine in this")
+                cosutil.printContinuation("build of CalCOS. Data may be calibrated with an incorrect")
+                cosutil.printContinuation("Y walk correction, potentially resulting in incorrect")
+                cosutil.printContinuation("spectral extraction.")
+            ycorrection = walkCorrection(events.field('ycorr'),
                                          events.field('pha'),
                                          reffiles["ywlkfile"],
                                          info["segment"])
@@ -2120,6 +2129,19 @@ def doYWalkcorr(events, info, switches, reffiles, phdr):
         else:
             return None
 
+
+def YWalkReffile_hasWrongSize(ywalkfile, segment, correct_size):
+    fd = fits.open(ywalkfile)
+    for extension in fd[1:]:
+        if extension.header['SEGMENT'] == segment:
+            reference_array = extension.data
+            break
+    reference_data_shape = reference_array.shape
+    fd.close()
+    return reference_data_shape != correct_size
+    pass
+
+
 def walkCorrection(fastCoordinate, slowCoordinate, reference_file, segment):
     """Apply walk correction
     The same algorithm is used for both.
@@ -2128,14 +2150,14 @@ def walkCorrection(fastCoordinate, slowCoordinate, reference_file, segment):
 
     Parameters
     ----------
-    slowCoordinate: numpy ndarray
-        The array of coordinates that is used to look up the correction in the
-        slow direction of the reference array
-
     fastCoordinate: numpy ndarray
         The array of coordinates that is used to look up the correction in the
         fast direction of the reference array
 
+    slowCoordinate: numpy ndarray
+        The array of coordinates that is used to look up the correction in the
+        slow direction of the reference array
+
     reference_file: string
         Name of reference file
 
@@ -2158,6 +2180,7 @@ def walkCorrection(fastCoordinate, slowCoordinate, reference_file, segment):
     delta = np.zeros(len(fastCoordinate))
     delta = bilinear_interpolation(fastCoordinate, slowCoordinate,
                            reference_array)
+    fd.close()
     return delta
 
 def bilinear_interpolation(fastCoordinate, slowCoordinate,
@@ -3132,6 +3155,76 @@ def doFlatcorr(events, info, switches, reffiles, phdr, hdr):
 
         phdr["flatcorr"] = "COMPLETE"
 
+def doHvdscorr(events, info, switches, reffiles, phdr, hdr):
+    """Apply High Voltage Sensitivity Dependence Correction.
+
+    Parameters
+    ----------
+    events : astropy.io.fits record array
+        The data unit containing the events table.
+
+    info : dictionary
+        Header keywords and values.
+
+    switches : dictionary
+        Calibration switches.
+
+    reffiles : dictionary
+        Reference file names.
+
+    phdr : astropy.io.fits Header object
+        The input primary header.
+
+    hdr : astropy.io.fits Header object
+        The events extension header.
+    """
+
+    cosutil.printSwitch("HVDSCORR", switches)
+
+    if switches["hvdscorr"] == "PERFORM":
+
+        cosutil.printRef("HVDSTAB", reffiles)
+
+        if info["detector"] == "FUV":
+            segment = info['segment']
+            slope, intercept, zeropoint = getHVDSParamsfromReffile(phdr, reffiles)
+            hvkeyword = 'hvlevel' + segment[-1]
+            hv = hdr[hvkeyword]
+            epsmultiplier = intercept + slope*(hv - zeropoint)
+            epsilon = events.field("epsilon")
+            epsilon *= epsmultiplier
+            events['epsilon'][:] = epsilon
+
+        phdr["hvdscorr"] = "COMPLETE"
+
+def getHVDSParamsfromReffile(info, reffiles):
+    """Get straight line fit parameters from HVDSTAB reference file.
+
+    Parameters
+    ----------
+    info : dictionary
+        Header keywords and values.
+
+    reffiles : dictionary
+        Reference file names.
+
+    Returns
+    -------
+    (slope, intercept, zeropoint) : tuple of floats
+        Slope, intercept and zeropoint fit parameters from the reference file
+
+    """
+
+    segment = info['segment']
+    cenwave = info['cenwave']
+    filter = {"segment": segment, 'cenwave': cenwave}
+    hvdsrow = cosutil.getTable(reffiles["hvdstab"], filter, exactly_one=True,
+                               at_least_one=True)
+    slope = hvdsrow['slope']
+    intercept = hvdsrow['intercept']
+    zeropoint = hvdsrow['zeropoint']
+    return (slope, intercept, zeropoint)
+
 def convolveFlat(flat, dispaxis,
                  expstart, exptime, dopmagt, dopzerot, orbtpert):
     """Convolve the flat field file with the Doppler smearing function.