Merge pull request #179 from sdss/fixhdr

Fixing missing/redundant header information

python/lvmdrp/core/image.py

@@ -1124,22 +1124,23 @@ def convertUnit(self, to, assume="adu", gain_field="GAIN", inplace=False):
             return new_image
 
         if current != to:
+            camera = self.getHdrValue("CCD").upper()
             exptime = self.getHdrValue("EXPTIME")
-            gains = self.getHdrValue(f"AMP? {gain_field}")
-            sects = self.getHdrValue("AMP? TRIMSEC")
+            gains = self.getHdrValue(f"{camera} AMP? {gain_field}")
+            sects = self.getHdrValue(f"{camera} AMP? TRIMSEC")
             n_amp = len(gains)
             for i in range(n_amp):
                 if current == "adu" and to == "electron":
                     factor = gains[i]
-                elif current == "adu" and to == "electron/s":
+                elif current == "adu" and to == "electron / s":
                     factor = gains[i] / exptime
                 elif current == "electron" and to == "adu":
                     factor = 1 / gains[i]
-                elif current == "electron" and to == "electron/s":
+                elif current == "electron" and to == "electron / s":
                     factor = 1 / exptime
-                elif current == "electron/s" and to == "adu":
+                elif current == "electron / s" and to == "adu":
                     factor = gains[i] * exptime
-                elif current == "electron/s" and to == "electron":
+                elif current == "electron / s" and to == "electron":
                     factor = exptime
                 else:
                     raise ValueError(f"Cannot convert from {current} to {to}")
@@ -1343,7 +1344,7 @@ def loadFitsData(
                     elif hdu[i].header["EXTNAME"].split()[0] == "FRAMES":
                         self._individual_frames = Table(hdu[i].data)
                     elif hdu[i].header["EXTNAME"].split()[0] == "SLITMAP":
-                        self._slitmap = Table(hdu[i].data)
+                        self._slitmap = Table.read(hdu[i])
 
         else:
             if extension_data is not None:
@@ -1360,7 +1361,7 @@ def loadFitsData(
             if extension_frames is not None:
                 self._individual_frames = Table(hdu[extension_frames].data)
             if extension_slitmap is not None:
-                self._slitmap = Table(hdu[extension_slitmap].data)
+                self._slitmap = Table.read(hdu[extension_slitmap])
 
         # set is_masked attribute
         self.is_masked = numpy.isnan(self._data).any()
@@ -1454,15 +1455,12 @@ def writeFitsData(
         #    hdus[0].update_ext_name('T')
 
         if len(hdus) > 0:
-            hdus[0].header['DRPVER'] = drpver
             hdu = pyfits.HDUList(hdus)  # create an HDUList object
             if self._header is not None:
-                hdu[0].header = self.getHeader()  # add the primary header to the HDU
-                try:
-                    hdu[0].header["BZERO"] = 0
-                except KeyError:
-                    pass
+                hdu[0].header = self.getHeader()
+                hdu[0].header['DRPVER'] = drpver
                 hdu[0].update_header()
+            hdu[0].scale(bzero=0, bscale=1)
 
         os.makedirs(os.path.dirname(filename), exist_ok=True)
         hdu.writeto(filename, output_verify="silentfix", overwrite=True)
@@ -3154,7 +3152,7 @@ def getSlitmap(self):
 
     def setSlitmap(self, slitmap):
         if isinstance(slitmap, pyfits.BinTableHDU):
-            self._slitmap = Table(slitmap.data)
+            self._slitmap = Table.read(slitmap)
         else:
             self._slitmap = slitmap
 
@@ -3376,7 +3374,8 @@ def combineImages(
     stack_error[stack_mask] = numpy.nan
 
     if background_subtract:
-        quad_sections = images[0].getHdrValues("AMP? TRIMSEC")
+        camera = images[0].getHdrValue("CCD").upper()
+        quad_sections = images[0].getHdrValues(f"{camera} AMP? TRIMSEC")
         stack_image, _, _, _ = _bg_subtraction(
             images=stack_image,
             quad_sections=quad_sections,

python/lvmdrp/core/plot.py

@@ -271,7 +271,8 @@ def plot_detrend(ori_image, det_image, axs, mbias=None, mdark=None, labels=False
     """
 
     # define quadrant sections
-    sections = ori_image.getHdrValue("AMP? TRIMSEC")
+    camera = ori_image.getHdrValue("CCD").upper()
+    sections = ori_image.getHdrValue(f"{camera} AMP? TRIMSEC")
 
     # convert all images to adu
     unit = "adu"

python/lvmdrp/core/sky.py

@@ -246,9 +246,9 @@ def skymodel_pars_header(header):
     skymodel_pars: dict
         output dict with header keywords, values and comments
         None/NAN/Null values replaced with -999
-    
+
     """
-    
+
     # extract useful header information
     sci_ra, sci_dec = header["TESCIRA"], header["TESCIDE"]
     skye_ra, skye_dec = header["TESKYERA"], header["TESKYEDE"]
@@ -325,7 +325,7 @@ def skymodel_pars_header(header):
 
     # Moon illumination fraction
     moon_fli = m.fraction_illuminated(sun)
- 
+
     # altitude of sun ('altsun', -90 -- 90)
     altsun, _, _ = s.altaz()
 
@@ -387,37 +387,37 @@ def skymodel_pars_header(header):
 
 
     skymodel_pars = {
-        "HIERARCH SKYMODEL SM_H": (sm_h.to(u.km).value, "observatory height in km"),
-        "HIERARCH SKYMODEL SM_HMIN": ((2.0 * u.km).value, "lower height limit in km"),
-        "HIERARCH SKYMODEL SCI_ALT": (np.round(sci_alt.to(u.deg).value, 4), "altitude of object above horizon (deg)"),
-        "HIERARCH SKYMODEL SKYE_ALT": (np.round(skye_alt.to(u.deg).value, 4), "altitude of object above horizon (deg)"),
-        "HIERARCH SKYMODEL SKYW_ALT": (np.round(skyw_alt.to(u.deg).value, 4), "altitude of object above horizon (deg)"),
-        "HIERARCH SKYMODEL ALPHA": (np.round(alpha.to(u.deg).value, 4), "separation of Sun and Moon from Earth (deg)"),
-        "HIERARCH SKYMODEL SCI_RHO": (np.round(sci_rho.to(u.deg).value, 4), "separation of Moon and object (deg)"),
-        "HIERARCH SKYMODEL SKYE_RHO": (np.round(skye_rho.to(u.deg).value, 4), "separation of Moon and object (deg)"),
-        "HIERARCH SKYMODEL SKYW_RHO": (np.round(skyw_rho.to(u.deg).value, 4), "separation of Moon and object (deg)"),
-        "HIERARCH SKYMODEL MOONALT": (np.round(altmoon.to(u.deg).value, 4), "altitude of Moon above horizon (deg)"),
-        "HIERARCH SKYMODEL SUNALT": (np.round(altsun.to(u.deg).value, 4), "altitude of Sun above horizon (deg)"),
+        "HIERARCH SKYMODEL SM_H": (sm_h.to(u.km).value, "observatory height [km]"),
+        "HIERARCH SKYMODEL SM_HMIN": ((2.0 * u.km).value, "lower height limit [km]"),
+        "HIERARCH SKYMODEL SCI_ALT": (np.round(sci_alt.to(u.deg).value, 4), "altitude of object above horizon [deg]"),
+        "HIERARCH SKYMODEL SKYE_ALT": (np.round(skye_alt.to(u.deg).value, 4), "altitude of object above horizon [deg]"),
+        "HIERARCH SKYMODEL SKYW_ALT": (np.round(skyw_alt.to(u.deg).value, 4), "altitude of object above horizon [deg]"),
+        "HIERARCH SKYMODEL ALPHA": (np.round(alpha.to(u.deg).value, 4), "separation of Sun and Moon from Earth [deg]"),
+        "HIERARCH SKYMODEL SCI_RHO": (np.round(sci_rho.to(u.deg).value, 4), "separation of Moon and object [deg]"),
+        "HIERARCH SKYMODEL SKYE_RHO": (np.round(skye_rho.to(u.deg).value, 4), "separation of Moon and object [deg]"),
+        "HIERARCH SKYMODEL SKYW_RHO": (np.round(skyw_rho.to(u.deg).value, 4), "separation of Moon and object [deg]"),
+        "HIERARCH SKYMODEL MOONALT": (np.round(altmoon.to(u.deg).value, 4), "altitude of Moon above horizon [deg]"),
+        "HIERARCH SKYMODEL SUNALT": (np.round(altsun.to(u.deg).value, 4), "altitude of Sun above horizon [deg]"),
         "HIERARCH SKYMODEL MOONDIST": (np.round(moondist.value, 4), "distance to Moon (mean distance=1)"),
-        "HIERARCH SKYMODEL PRES": ((744 * u.hPa).value, "pressure at observer altitude (hPa), set: 744"),
+        "HIERARCH SKYMODEL PRES": ((744 * u.hPa).value, "pressure at observer altitude, set: 744 [hPa]"),
         "HIERARCH SKYMODEL SSA": (0.97, "aerosols' single scattering albedo, set: 0.97"),
-        "HIERARCH SKYMODEL CALCDS": ( "N", "cal double scattering of moon (Y or N)"),
-        "HIERARCH SKYMODEL O2COLUMN": (1.0, "relative ozone column density (1->258 DU)"),
+        "HIERARCH SKYMODEL CALCDS": ( "N", "cal double scattering of Moon (Y or N)"),
+        "HIERARCH SKYMODEL O2COLUMN": (1.0, "relative ozone column density (1->258) [DU]"),
         "HIERARCH SKYMODEL MOONSCAL": (1.0, "scaling factor for scattered moonlight"),
-        "HIERARCH SKYMODEL SCI_LON_ECL": (np.round(sci_lon_ecl.to(u.deg).value, 5), "heliocen ecliptic longitude (deg)"),
-        "HIERARCH SKYMODEL SCI_LAT_ECL": (np.round(sci_lat_ecl.to(u.deg).value, 5), "ecliptic latitude (deg)"),
-        "HIERARCH SKYMODEL SKYE_LON_ECL": (np.round(skye_lon_ecl.to(u.deg).value, 5), "heliocen ecliptic longitude (deg)"),
-        "HIERARCH SKYMODEL SKYE_LAT_ECL": (np.round(skye_lat_ecl.to(u.deg).value, 5), "ecliptic latitude (deg)"),
-        "HIERARCH SKYMODEL SKYW_LON_ECL": (np.round(skyw_lon_ecl.to(u.deg).value, 5), "heliocen ecliptic longitude (deg)"),
-        "HIERARCH SKYMODEL SKYW_LAT_ECL": (np.round(skyw_lat_ecl.to(u.deg).value, 5), "ecliptic latitude (deg)"),
+        "HIERARCH SKYMODEL SCI_LON_ECL": (np.round(sci_lon_ecl.to(u.deg).value, 5), "heliocen ecliptic longitude [deg]"),
+        "HIERARCH SKYMODEL SCI_LAT_ECL": (np.round(sci_lat_ecl.to(u.deg).value, 5), "ecliptic latitude [deg]"),
+        "HIERARCH SKYMODEL SKYE_LON_ECL": (np.round(skye_lon_ecl.to(u.deg).value, 5), "heliocen ecliptic longitude [deg]"),
+        "HIERARCH SKYMODEL SKYE_LAT_ECL": (np.round(skye_lat_ecl.to(u.deg).value, 5), "ecliptic latitude [deg]"),
+        "HIERARCH SKYMODEL SKYW_LON_ECL": (np.round(skyw_lon_ecl.to(u.deg).value, 5), "heliocen ecliptic longitude [deg]"),
+        "HIERARCH SKYMODEL SKYW_LAT_ECL": (np.round(skyw_lat_ecl.to(u.deg).value, 5), "ecliptic latitude [deg]"),
         "HIERARCH SKYMODEL EMIS_STR": (0.2, "grey-body emissivity"),
-        "HIERARCH SKYMODEL TEMP_STR": ((290.0 * u.K).value, "grey-body temperature (K)"),
+        "HIERARCH SKYMODEL TEMP_STR": ((290.0 * u.K).value, "grey-body temperature [K]"),
         "HIERARCH SKYMODEL MSOLFLUX": (130.0, "monthly-averaged solar radio flux, set: 130"),
         "HIERARCH SKYMODEL SEASON": (season, "bimonthly period (1:Dec/Jan, 6:Oct/Nov.; 0:year)"),
         "HIERARCH SKYMODEL TIME": (time, "period of night (x/3 night, x=1,2,3; 0:night)"),
-        "HIERARCH SKYMODEL MOON_RA": (np.round(moonra.to(u.deg).value, 5), "RA of the moon (deg)"),
-        "HIERARCH SKYMODEL MOON_DEC": (np.round(moondec.to(u.deg).value, 5), "DEC of the moon (deg)"),
-        "HIERARCH SKYMODEL MOON_PHASE": (np.round(moon_phase, 2), "Phase of moon (deg; 0=new, 90=1st qt)"),
+        "HIERARCH SKYMODEL MOON_RA": (np.round(moonra.to(u.deg).value, 5), "RA of the Moon [deg]"),
+        "HIERARCH SKYMODEL MOON_DEC": (np.round(moondec.to(u.deg).value, 5), "DEC of the Moon [deg]"),
+        "HIERARCH SKYMODEL MOON_PHASE": (np.round(moon_phase, 2), "phase of Moon (0=new, 90=1st qt) [deg]"),
         "HIERARCH SKYMODEL MOON_FLI": (np.round(moon_fli, 4), "Moon fraction lunar illumination")
     }
 

python/lvmdrp/etc/dataproducts/calibration/lvmThroughput_bp.yaml

@@ -33,7 +33,7 @@ hdu1:
       name: IVAR
       type: float64
       unit: null
-      description: associated inverse variance (1/sigma^2) of the above
+      description: associated inverse variance of the above
     MASK:
       name: MASK
       type: int32

python/lvmdrp/etc/dataproducts/lvmCFrame_bp.yaml

@@ -1,16 +1,16 @@
 name: lvmCFrame
 short: the camera-combined science frames
 description: |
-    Per-exposure combinations of the lvmFrame files by stitching the blue, red, and nir channels
+    Per-exposure combinations of the lvmFFrame files by stitching the blue, red, and NIR channels
     together across the dichroic break, adding rows for fibers from spectrograph 2 and 3 atop
     those from spectrograph 1 (i.e., in order of fiberid). All spectra in this file have been
     resampled to a common wavelength grid for ALL LVM observations.  Note that 'C' in the
-    name means Spectrograph Combined.
+    name means Spectrograph Channel Combined.
 location: $LVM_SPECTRO_REDUX/[DRPVER]/[TILEID]/[MJD]/
 created_by: module.py
 used_by: module.py
 naming_convention: |
-  lvmCFrame-[CHANNEL]-[EXPNUM].fits.gz, where [CHANNEL] is the channel (brz) and [EXPNUM] is the (zero-padded) 8-digit exposure number.
+  lvmCFrame-[EXPNUM].fits.gz, where [EXPNUM] is the (zero-padded) 8-digit exposure number.
 hdu0:
   name: PRIMARY
   description: primary header metadata
@@ -21,56 +21,73 @@ hdu0:
     comment: conforms to FITS standard
 hdu1:
   name: FLUX
-  description: Flux in flat fielded electrons / angstrom
+  description: calibrated flux in erg / (Angstrom s cm2)
   is_image: true
+  shape: NWAVE x NFIBER
   header:
   - key: BUNIT
-    comment: flux units
-  - key: BSCALE
-    comment: linear scaling factor
-  - key: BZERO
-    comment: zero point
-  shape: NWAVE x NFIBER
+    comment: physical units of the array values
 hdu2:
   name: IVAR
-  description: Inverse variance (1/sigma2) for the above
+  description: inverse variance in cm4 Angstrom2 s2 / erg2
   is_image: true
   shape: NWAVE x NFIBER
+  header:
+  - key: BUNIT
+    comment: physical units of the array values
 hdu3:
   name: MASK
-  description: the quality bitmask for each fiber pixel
+  description: quality bitmask for each fiber pixel
   is_image: true
   shape: NWAVE x NFIBER
 hdu4:
   name: WAVE
-  description: air wavelength solution in angstroms
+  description: air wavelength solution in Angstrom
   is_image: true
   shape: NWAVE
+  header:
+  - key: BUNIT
+    comment: physical units of the array values
 hdu5:
   name: LSF
-  description: LSF (FWHM) solution in angstroms
+  description: LSF (FWHM) solution in Angstrom
   is_image: true
   shape: NWAVE x NFIBER
+  header:
+  - key: BUNIT
+    comment: physical units of the array values
 hdu6:
   name: SKY_EAST
-  description: sky east in flux-calibrated units (erg/s/cm2/Ang)
+  description: sky east in flux-calibrated units erg / (Angstrom s cm2)
   is_image: true
   shape: NWAVE x NFIBER
+  header:
+  - key: BUNIT
+    comment: physical units of the array values
 hdu7:
   name: SKY_EAST_IVAR
-  description: sky east inverse variance (1/sigma^2) in flux-calibrated units (erg/s/cm2/Ang)
+  description: sky east inverse variance in cm4 Angstrom2 s2 / erg2
   is_image: true
   shape: NWAVE x NFIBER
+  header:
+  - key: BUNIT
+    comment: physical units of the array values
 hdu8:
   name: SKY_WEST
-  description: sky west in flux-calibrated units (erg/s/cm2/Ang)
+  description: sky west in flux-calibrated units erg / (Angstrom s cm2)
   is_image: true
   shape: NWAVE x NFIBER
+  header:
+  - key: BUNIT
+    comment: physical units of the array values
 hdu9:
   name: SKY_WEST_IVAR
-  description: sky west inverse variance (1/sigma^2) in flux-calibrated units (erg/s/cm2/Ang)
+  description: sky west inverse variance in cm4 Angstrom2 s2 / erg2
   is_image: true
   shape: NWAVE x NFIBER
+  header:
+  - key: BUNIT
+    comment: physical units of the array values
 hdu10:
   name: SLITMAP
   description: slitmap table describing fiber positions for this exposure
@@ -156,17 +173,17 @@ hdu10:
       name: ypix_b
       type: uint32
       description: the y coordinate in pixels of the fiber at column 2000 for channel b
-      unit: pixels
+      unit: pix
     YPIX_R:
       name: ypix_r
       type: uint32
       description: the y coordinate in pixels of the fiber at column 2000 for channel r
-      unit: pixels
+      unit: pix
     YPIX_Z:
       name: ypix_z
       type: uint32
       description: the y coordinate in pixels of the fiber at column 2000 for channel z
-      unit: pixels
+      unit: pix
     FIBSTATUS:
       name: fibstatus
       type: uint16