Merge pull request #1 from hughbg/master

Pulling Hugh's code back in

01_plot_waterfall.py

@@ -5,80 +5,188 @@
 Plot data as calibrated waterfall plot.
 
 """
+import os
+import hickle
+from mpl_toolkits.mplot3d import Axes3D
+from matplotlib import cm
 import seaborn as sns
 import tables as tb
+import scipy.signal
 from leda_cal.skymodel import *
 from leda_cal.leda_cal import *
+from leda_cal.dpflgr import *
 
 sns.set_style('white')
 sns.set_context("poster",font_scale=.75)
 
-def quicklook(filename):
+ovro_location = ('37.2397808', '-118.2816819', 1183.4839)
+ovro = ephem.Observer(); (ovro.lat, ovro.lon, ovro.elev) = ovro_location
+sun = ephem.Sun()
+
+gal_center = ephem.FixedBody()  
+gal_center._ra  =  '17 45 40.04'
+gal_center._dec = '-29 00 28.1'
+gal_center.name = "Galactic Center"
+
+
+def quicklook(filename, save, dump, flag, no_show, all_lsts):
     h5 = tb.open_file(filename)
 
     T_ant = apply_calibration(h5)
     f_leda = T_ant['f']
 
-    ant_ids = ['252A', '254A', '255A']
-    pol_id  = 'y'
+    ant_ids = ['252', '254', '255']
 
     print("Plotting...")
-    fig, axes = plt.subplots(figsize=(12, 6), nrows=2, ncols=2)
+    fig, axes = plt.subplots(figsize=(12, 12))
     #plt.suptitle(h5.filename)
 
     lst_stamps = T_ant['lst']
+    # Report discontinuities in time
+    for i in range(1,len(lst_stamps)):
+      if lst_stamps[i]-lst_stamps[i-1] > 1/60.0:	# 1 minute
+        print "Discontinuity at LST", lst_stamps[i], (lst_stamps[i]-lst_stamps[i-1])*60*60, "seconds"
+
     utc_stamps = T_ant['utc']
     xlims = (f_leda[0], f_leda[-1])
     #ylims = mdates.date2num((T_ant['utc'][0], T_ant['utc'][-1]))
     #hfmt = mdates.DateFormatter('%m/%d %H:%M')
     ylims = (T_ant['lst'][0], T_ant['lst'][-1])
-    for ii, key in enumerate(ant_ids):
-        ax = fig.add_subplot(1, 3, ii+1)
-
-        T_flagged = T_ant[key]
-        #T_flagged = rfi_flag(T_ant[key], thr_f=0.07, thr_t=0.07, rho=1.5,
-        #         bp_window_f=8, bp_window_t=8, 
-        #         max_frac_f=0.3, max_frac_t=0.3)
+
+    # Work out altitude of Gal center and Sun. Use whichever is highest
+    # and put that in the padding, which is the stripe.
+    unusable_lsts = []
+    pad_length = 70
+    padding = np.zeros((len(lst_stamps), pad_length))
+    for i, d in enumerate(utc_stamps):
+      ovro.date = d
+      sun.compute(ovro)
+      gal_center.compute(ovro)
+      if sun.alt > -15*np.pi/180 or gal_center.alt > -15*np.pi/180:
+        padding[i, :] = 10000
+        unusable_lsts.append(i)
+      else: 
+        padding[i, :] = 1000
+
+    # Delete sun up LSTS
+    if not all_lsts:
+       padding = np.delete(padding, unusable_lsts, axis=0)
+       lst_stamps = np.delete(lst_stamps, unusable_lsts, axis=0)
+       utc_stamps = np.delete(utc_stamps, unusable_lsts, axis=0)
+       ylims = ( lst_stamps[0], lst_stamps[-1] )
+       print len(lst_stamps), "usable LSTs"
+    else: print "Using all LSTs"
+    if len(lst_stamps) == 0:
+      print "There is no data to display (number of LSTs is 0)"
+      exit(1) 
+
+    yloc = []
+    ylabel = []
+    for i in range(0, len(lst_stamps), len(lst_stamps)/7):
+      yloc.append(lst_stamps[i]), ylabel.append(("%.1f" % lst_stamps[i]))
+    if all_lsts: new_x_high = xlims[1]+pad_length*(xlims[1]-xlims[0])/len(f_leda)
+    else: new_x_high = xlims[1]
+
+    dump_data = {}
+
+    for p in [ 0, 1 ]:
+
+      for ii, key in enumerate(ant_ids):
+          ax = fig.add_subplot(2, 3, 3*p+ii+1)
 
-        T_ant[key] = T_flagged
+	  if p == 0: ant = key+"A"
+          else: ant = key+"B"
+
+          T_flagged = T_ant[ant]
+          if not all_lsts:  T_flagged = np.delete(T_flagged, unusable_lsts, axis=0)
+
+          if flag:
+            T_flagged = rfi_flag(T_flagged, thr_f=0.2, thr_t=0.2, rho=1.5,
+               bp_window_f=16, bp_window_t=16,
+               max_frac_f=0.5, max_frac_t=0.5)
+
+
+
+          if dump: dump_data[ant] = T_flagged
+          if flag: 
+            total = T_flagged.shape[0]*T_flagged.shape[1]
+            num_in = np.ma.MaskedArray.count(T_flagged)
+            print ant, ( "%.1f%%" % (100*(total-num_in)/total) ), "flagged.", "Count:", total-num_in
 
-        im = plt.imshow(T_flagged, # / np.median(xx, axis=0), 
+          # Add the stripe onto the right edge of the data and adjust the extent of the x-axis (frequency) to cover the stripe.
+          if all_lsts: T_flagged_plot = np.ma.concatenate((T_flagged, padding), axis=1)
+          else: T_flagged_plot = T_flagged
+
+          axim = plt.subplot(2, 3, 3*p+ii+1)
+          axim.set_yticks(yloc)
+          axim.set_yticklabels(ylabel)
+	  axim.tick_params(axis='y', pad=2)
+
+          im = plt.imshow(T_flagged_plot, # / np.median(xx, axis=0), 
                    cmap='viridis', aspect='auto',
                    interpolation='nearest',
                    clim=(1000, 10000),
-                   extent=(xlims[0], xlims[1], ylims[1], ylims[0])
+                   extent=(xlims[0], new_x_high, ylims[1], ylims[0])
                    )
-        plt.title(ant_ids[ii])
-        plt.xlabel("Frequency [MHz]")
-        #ax.yaxis_date()
-        #ax.yaxis.set_major_formatter(hfmt)
-        #
 
-
-    plt.subplot(1,3,1)
+          plt.title(ant)
+ 	  if p == 1: plt.xlabel("Frequency [MHz]")
+          #ax.yaxis_date()
+          #ax.yaxis.set_major_formatter(hfmt)
+          #
+
+    plt.subplot(2,3,1)
+    plt.ylabel("LST [hr]")
+    plt.subplot(2,3,4)
     plt.ylabel("LST [hr]")
+
 
-    fig.subplots_adjust(left=0.06)
+    fig.subplots_adjust(left=0.07)
     fig.subplots_adjust(right=0.875)
     cbar_ax = fig.add_axes([0.9, 0.125, 0.025, 0.75])
     cbar = fig.colorbar(im, cax=cbar_ax)
 
-    plt.subplot(1,3,3)
+    #plt.subplot(2,3,3)
     #cbar = plt.colorbar()
     cbar.set_label("Temperature [K]")
-
+    cbar.ax.tick_params(axis='y', pad=2) 
     #plt.tight_layout()
-    plt.show()
 
+    if save:
+      plt.savefig(os.path.basename(filename)[:-3]+".png")
+    if not no_show:
+      plt.show()
 
+    if dump:
+      dump_data["lsts"] = lst_stamps
+      dump_data["utcs"] = np.array([str(pytime) for pytime in utc_stamps])
+      dump_data["frequencies"] = f_leda
+      dump_data["options"] = "Flag="+str(flag)+" All LSTSs="+str(all_lsts)
+      hickle.dump(dump_data, os.path.basename(filename)[:-3]+".hkl")
 
 if __name__ == "__main__":
+    import optparse, sys
+
+    usage = '%prog [opts] filename_of_hdf5_observation'
+    o = optparse.OptionParser()
+    o.set_usage(usage)
+    o.set_description(__doc__)
+    o.add_option('--flag', dest='flag', action='store_true', default=False,
+      help='Apply flagging. Default: False')
+    o.add_option('--all_lsts', dest='all_lsts', action='store_true', default=False,
+      help='Include all LSTs, not just when Galaxy and Sun are down. A day/night stripe is printed on the right. Default: False.')
+    o.add_option('--dump', dest='dump', action='store_true', default=False,
+      help='Dump the data to a hickle file, with filename the same as the h5 but hkl extension. Default: False.')
+    o.add_option('--save', dest='save', action='store_true', default=False,
+      help="Save the plot to an image file, with filename the same as the h5 but png extension. Default: False.")
+    o.add_option('--no_show', dest='no_show', action='store_true', default=False,
+      help="Don't display the plot on screen. Useful for batch runs. Default: False.")
+
+    opts, args = o.parse_args(sys.argv[1:])
 
-    import sys
-    try:
-        filename = sys.argv[1]
-    except:
-        print "USAGE: ./quicklook.py filename_of_hdf5_observation"
-        exit()
-
-    quicklook(filename)
+    if len(args) != 1:
+      o.print_help()
+      exit(1)
+    else: filename = args[0]
+
+    quicklook(filename, opts.save, opts.dump, opts.flag, opts.no_show, opts.all_lsts)

02_plot_spectra.py

@@ -5,6 +5,7 @@
 Plots the time-averaged spectra.
 """
 import matplotlib as mpl
+import os
 import seaborn as sns
 import tables as tb
 from leda_cal.leda_cal import *
@@ -13,15 +14,15 @@
 sns.set_style('ticks')
 sns.set_context("paper",font_scale=1.5)
 
-def quicklook(filename):
+def quicklook(filename, save, noshow):
     h5 = tb.open_file(filename)
 
     T_ant = apply_calibration(h5)
     f_leda = T_ant['f']
 
     print T_ant.keys()
 
-    ant_ids = ['252A', '254A', '255A', '254B', '255B']
+    ant_ids = ['252A', '254A', '255A', '252B', '254B', '255B'] # Added 252B, why was it missing? HG
     pol_id  = 'y'
 
     print("Plotting...")
@@ -49,6 +50,8 @@ def quicklook(filename):
     plt.plot(0, 0)
     plt.plot(f_leda, np.median(T_ant[ant_ids[3]][mid-sl:mid+sl], axis=0), label=ant_ids[3])
     plt.plot(f_leda, np.median(T_ant[ant_ids[4]][mid-sl:mid+sl], axis=0), label=ant_ids[4])
+    plt.plot(f_leda, np.median(T_ant[ant_ids[5]][mid-sl:mid+sl], axis=0), label=ant_ids[5])
+
     plt.legend(frameon=False)
 
     plt.xlim(40, 85)
@@ -62,16 +65,29 @@ def quicklook(filename):
     plt.legend(frameon=False)
     plt.tight_layout()
     plt.savefig("figures/compare-spectra.pdf")
-    plt.show()
 
+    if save:
+      plt.savefig("spec_"+os.path.basename(filename)[:-3]+".png")
+
+    if not noshow: plt.show()
 
 if __name__ == "__main__":
-
-    import sys
-    try:
-        filename = sys.argv[1]
-    except:
-        print "USAGE: ./quicklook.py filename_of_hdf5_observation"
-        exit()
-
-    quicklook(filename)
+    import optparse, sys
+
+    usage = '%prog [opts] filename_of_hdf5_observation'
+    o = optparse.OptionParser()
+    o.set_usage(usage)
+    o.set_description(__doc__)
+    o.add_option('--save', dest='save', action='store_true', default=False,
+      help='Save plot to a file. Default: False')
+    o.add_option('--noshow', dest='noshow', action='store_true', default=False,
+      help="Don't display the plot on the screen. Default: False")
+    opts, args = o.parse_args(sys.argv[1:])
+
+    if len(args) != 1:
+      o.print_help()
+      exit(1)
+    else: filename = args[0]
+
+    quicklook(filename, opts.save, opts.noshow)
+

03_compare_waterfall.py

@@ -7,7 +7,7 @@
 """
 import seaborn as sns
 import tables as tb
-from skymodel import *
+from leda_cal.skymodel import *
 from leda_cal.leda_cal import *
 sns.set_style('white')
 sns.set_context("poster",font_scale=.75)
@@ -71,7 +71,7 @@ def quicklook(filename):
 
     #plt.tight_layout()
     plt.show()
-
+    exit()	# I get an error if it goes further
     plt.xlabel("Frequency [MHz]")
 
     plt.ylabel("LST [hr]")
@@ -100,4 +100,4 @@ def quicklook(filename):
         print "USAGE: ./quicklook.py filename_of_hdf5_observation"
         exit()
 
-    quicklook(filename)
+    quicklook(filename)

04_compare_spectra.py

@@ -9,7 +9,7 @@
 
 import seaborn as sns
 import tables as tb
-from skymodel import *
+from leda_cal.skymodel import *
 from leda_cal.leda_cal import *
 from leda_cal.dpflgr import simple_flag
 sns.set_style('ticks')
@@ -112,4 +112,4 @@ def quicklook(filename):
         print "USAGE: ./quicklook.py filename_of_hdf5_observation"
         exit()
 
-    quicklook(filename)
+    quicklook(filename)

06_plot_rfi.py

@@ -97,4 +97,4 @@ def quicklook(filename):
         print "USAGE: ./quicklook.py filename_of_hdf5_observation"
         exit()
 
-    quicklook(filename)
+    quicklook(filename)

12_skymodel_compare.py

@@ -45,7 +45,7 @@ def quicklook(filename):
 
 
     import hickle as hkl
-    gsm = hkl.load("gsm-spec-lst11.hkl")
+    gsm = hkl.load("cal_data/gsm-spec-lst11.hkl")
     plt.plot(gsm["f"], gsm["T_ew"], c='#333333', ls='dashed')
 
     d = T_flagged[mid-sl:mid+sl].mean(axis=0)
@@ -89,4 +89,4 @@ def quicklook(filename):
         print "USAGE: ./quicklook.py filename_of_hdf5_observation"
         exit()
 
-    quicklook(filename)
+    quicklook(filename)

13_subtract_skymodel.py

@@ -55,7 +55,7 @@ def quicklook(filename):
 
 
     import hickle as hkl
-    gsm = hkl.load("gsm-spec-lst11.hkl")
+    gsm = hkl.load("cal_data/gsm-spec-lst11.hkl")
     #plt.plot(gsm["f"], gsm["T_ew"], c='#333333', ls='dashed')
 
     d = T_flagged[mid-sl:mid+sl].mean(axis=0)
@@ -133,4 +133,4 @@ def quicklook(filename):
         print "USAGE: ./quicklook.py filename_of_hdf5_observation"
         exit()
 
-    quicklook(filename)
+    quicklook(filename)

13b_subtract_skymodel.py

@@ -41,7 +41,7 @@ def quicklook(filename):
 
 
     import hickle as hkl
-    gsm = hkl.load("gsm-spec-lst11.hkl")
+    gsm = hkl.load("cal_data/gsm-spec-lst11.hkl")
     sl  = 250    
     n_poly = 5
     n_chan = 42
@@ -127,4 +127,4 @@ def quicklook(filename):
         print "USAGE: ./quicklook.py filename_of_hdf5_observation"
         exit()
 
-    quicklook(filename)
+    quicklook(filename)