draft nuv dark monitor. help.

cosmo/monitors/data_models.py

@@ -217,12 +217,13 @@ def get_new_data(self):
                 'YCORR'
                 ],
             3: [
+                'TIME',
                 'LATITUDE',
-                'LONGITUDE'
+                'LONGITUDE',
+                'DARKRATE'
                 ]
 
             }
-        # TODO: add gross counts, don't need PHA, lat or long right now
 
         # any special data requests
         # TODO: add spt support for temp, sun_lat, sun_long
@@ -235,7 +236,6 @@ def get_new_data(self):
         for program in program_ids:
             new_files_source = os.path.join(FILES_SOURCE, program)
             subfiles = glob(os.path.join(new_files_source, "*corrtag*"))
-            # TODO: figure out why this wasn't working with find_files()
             files += subfiles
 
         if not files:  # No new files
@@ -247,3 +247,61 @@ def get_new_data(self):
                                            table_request=table_request)
 
         return data_results
+
+
+class CorrtagDataModel(BaseDataModel):
+    """Datamodel for all NUV Dark files."""
+    files_source = FILES_SOURCE
+    subdir_pattern = '?????'
+
+    def get_new_data(self):
+        header_request = {
+            0: ['ROOTNAME'],
+            1: [
+                'EXPSTART',
+                'EXPTIME',
+                'EXPTYPE',
+                'DETECTOR'
+                ]
+            }
+
+        table_request = {
+            1: [
+                'TIME',
+                'XCORR',
+                'YCORR'
+                ],
+            3: [
+                'LATITUDE',
+                'LONGITUDE'
+                ]
+
+            }
+        # TODO: add gross counts, don't need PHA, lat or long right now
+
+        # any special data requests
+        # TODO: add spt support for temp, sun_lat, sun_long
+        # TODO: is this a good place to add solar data scraping in the future?
+
+        # this is temporary to find the files from the dark programs until
+        # we can add the dark files to the monitor_data database
+        # files = []
+        # program_ids = ['15776/']
+        # for program in program_ids:
+        #     new_files_source = os.path.join(FILES_SOURCE, program)
+        #     subfiles = glob(os.path.join(new_files_source, "*corrtag*"))
+        #     # TODO: figure out why this wasn't working with find_files()
+        #     files += subfiles
+
+        files = find_files('*corrtag*', data_dir=self.files_source,
+                           subdir_pattern=self.subdir_pattern)
+
+        if not files:  # No new files
+            return pd.DataFrame()
+
+        # need to add any other keywords that need to be set
+        data_results = data_from_exposures(files,
+                                           header_request=header_request,
+                                           table_request=table_request)
+
+        return data_results

cosmo/monitors/nuv_dark_monitor.py

@@ -28,24 +28,29 @@ def get_data(self):
         # access data, perform any filtering required for analysis
         data = self.model.new_data
         dark_rate_column = []
+        dec_year_column = []
 
         xlim = [0, 1024]
         ylim = [0, 1024]
 
         # parallelize, this is going to get bad when looking at a lot of data
         for index, row in data.iterrows():
             subdf = pd.DataFrame({
-                "TIME": row["TIME"], "XCORR": row["XCORR"],
-                "YCORR": row["YCORR"]
+                "EXPSTART": row["EXPSTART"], "TIME": row["TIME"],
+                "XCORR": row["XCORR"], "YCORR": row["YCORR"],
+                "TIME_3": row["TIME_3"]
                 })
             filtered = subdf.where(
                 (subdf["XCORR"] > xlim[0]) & (subdf["XCORR"] < xlim[1]) & (
                             subdf["YCORR"] > ylim[0]) & (
                             subdf["YCORR"] < ylim[1]))
-            dark_rate = self.calculate_dark_rate(filtered, xlim, ylim)
-            dark_rate_column.append(dark_rate)
+            dark_rate_array, dec_year_array = self.calculate_dark_rate(
+                filtered, xlim, ylim)
+            dark_rate_column.append(dark_rate_array)
+            dec_year_column.append(dec_year_array)
 
         data["DARK_RATE"] = dark_rate_column
+        data["DECIMAL_YEAR"] = dec_year_column
 
         return data
 
@@ -59,19 +64,33 @@ def calculate_dark_rate(self, dataframe, xlim, ylim):
 
         counts = np.histogram(dataframe["TIME"], bins=time_bins)[0]
         npix = float((xlim[1] - xlim[0]) * (ylim[1] - ylim[0]))
-        dark_rate = np.median(counts / npix / timestep)
-        # what is going on with the whole histogram, ok to use median?
-
-        return dark_rate
+        dark_rate_array = counts / npix / timestep
+        # save the whole histogram in time bins, and then plot each of them
+
+        # make a decimal year array corresponding to the time bins of the
+        # dark rates
+        # do this with the expstart (mjd) and time array from the timeline
+        # extension
+        mjd_conversion_factor = 1.15741e-5
+        # taking the expstart, binning the time array by the timestep,
+        # removing the last element in the array (bin by front edge),
+        # and then multiplying by the conversion factor
+        mjd_array = dataframe["EXPSTART"] + dataframe["TIME_3"][::timestep][
+                                            :-1] * mjd_conversion_factor
+        mjd_array = Time(mjd_array, format='mjd')
+        dec_year_array = mjd_array.decimalyear
+
+        return dark_rate_array, dec_year_array
 
     def track(self):
         # track something. perhaps current dark rate?
         pass
 
     def plot(self):
         # do some plots
+
         pass
 
     def store_results(self):
         # need to store results if not going in the database
-        pass
+        pass