From 6109337eac9cf7252f93aebff96fb5edc285c11f Mon Sep 17 00:00:00 2001 From: Ellie McLaurin Date: Mon, 29 Aug 2022 16:04:51 -0600 Subject: [PATCH 1/2] added temp_disc and edited existing code for testing --- notebooks/data_inspect.ipynb | 283 +++++++++++++++++++++++++++++++++-- 1 file changed, 273 insertions(+), 10 deletions(-) diff --git a/notebooks/data_inspect.ipynb b/notebooks/data_inspect.ipynb index 50d40eb..9b917ca 100755 --- a/notebooks/data_inspect.ipynb +++ b/notebooks/data_inspect.ipynb @@ -2,11 +2,24 @@ "cells": [ { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": { "scrolled": false }, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "import matplotlib\n", "import matplotlib.pyplot as plt\n", @@ -33,6 +46,10 @@ "from scipy import stats\n", "from IPython.display import display, HTML\n", "#warnings.filterwarnings('ignore')\n", + "#My changes:\n", + "from matplotlib import cm\n", + "from time import gmtime, strftime\n", + "from hera_mc import cm_active\n", "\n", "%matplotlib inline\n", "%config InlineBackend.figure_format = 'retina'\n", @@ -41,18 +58,29 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "metadata": { "scrolled": false }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "DATA_PATH = \"/lustre/aoc/projects/hera/dstorer/H5C_data/forEllie/2459660\"\n", + "APRIORI_STATUSES = dish_maintenance,dish_ok,RF_maintenance,RF_ok,digital_ok,digital_maintenance,calibration_maintenance,calibration_triage,calibration_ok\n", + "JULIANDATE = 2459660\n", + "Date = 3-21-2022\n" + ] + } + ], "source": [ "#get data location\n", - "data_path = os.environ['DATA_PATH']\n", + "data_path = '/lustre/aoc/projects/hera/dstorer/H5C_data/forEllie/2459660'\n", "print(f'DATA_PATH = \"{data_path}\"')\n", - "statuses = os.environ['APRIORI_STATUSES']\n", + "statuses = 'dish_maintenance,dish_ok,RF_maintenance,RF_ok,digital_ok,digital_maintenance,calibration_maintenance,calibration_triage,calibration_ok'\n", "print(f'APRIORI_STATUSES = {statuses}')\n", - "JD = os.environ['JULIANDATE']\n", + "JD = 2459660\n", "print(f'JULIANDATE = {JD}')\n", "utc = Time(JD, format='jd').datetime\n", "print(f'Date = {utc.month}-{utc.day}-{utc.year}')" @@ -64,9 +92,22 @@ "metadata": { "scrolled": true }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "21:55:14\n", + "755 sum files found between JDs 2459660.50018 and 2459660.66885\n", + "755 diff files found between JDs 2459660.50018 and 2459660.66885\n", + "755 sum auto files found between JDs 2459660.50018 and 2459660.66885\n", + "755 diff auto files found between JDs 2459660.50018 and 2459660.66885\n" + ] + } + ], "source": [ "# Load in data\n", + "print(strftime(\"%H:%M:%S\", gmtime()))\n", "HHfiles, difffiles, HHautos, diffautos, uvdx, uvdy = utils.load_data(data_path,JD)\n", " \n", "uvd = UVData()\n", @@ -84,7 +125,8 @@ "uvd.read(HHautos[::10], skip_bad_files=True, antenna_nums = use_ants)\n", "lsts = uvd.lst_array\n", "uvdx.select(antenna_nums=use_ants)\n", - "uvdy.select(antenna_nums=use_ants)" + "uvdy.select(antenna_nums=use_ants)\n", + "print(strftime(\"%H:%M:%S\", gmtime()))" ] }, { @@ -400,6 +442,227 @@ "utils.plot_wfs(uvd,1,mean_sub=True,jd=JD)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Temp Disc Plots" + ] + }, + { + "cell_type": "raw", + "metadata": {}, + "source": [ + "decription" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "generate node dict" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def generate_nodeDict(uv,pol='E'):\n", + " \"\"\"\n", + " Generates dictionaries containing node and antenna information.\n", + " \n", + " Parameters:\n", + " ----------\n", + " uv: UVData Object\n", + " Sample observation to extract node and antenna information from.\n", + " \n", + " Returns:\n", + " -------\n", + " nodes: Dict\n", + " Dictionary containing entry for all nodes, each of which has keys: 'ants', 'snapLocs', 'snapInput'.\n", + " antDict: Dict\n", + " Dictionary containing entry for all antennas, each of which has keys: 'node', 'snapLocs', 'snapInput'.\n", + " inclNodes: List\n", + " Nodes that have hooked up antennas.\n", + " \"\"\"\n", + " \n", + " antnums = uv.get_ants()\n", + " h = cm_hookup.Hookup()\n", + " x = h.get_hookup('HH')\n", + " nodes = {}\n", + " antDict = {}\n", + " inclNodes = []\n", + " for ant in antnums:\n", + " key = 'HH%i:A' % (ant)\n", + " n = x[key].get_part_from_type('node')[f'{pol}maxants:\n", + " maxants = n\n", + " \n", + " Nants = len(ants)\n", + " Nside = maxants\n", + " Yside = len(inclNodes)\n", + " \n", + "\n", + " for i,n in enumerate(inclNodes):\n", + " ants = nodes[n]['ants']\n", + " j = 0\n", + " for _,a in enumerate(sorted_ants):\n", + " if a not in ants:\n", + " continue\n", + " snaploc = antDict[a]['snapLocs']\n", + " snapinput = antDict[a]['snapInput']\n", + " fig = plt.figure(constrained_layout=True, figsize=(10, 4))\n", + " subfigs = fig.subfigures(1, 2, wspace=0.07, width_ratios=[3, 1])\n", + " axsLeft = subfigs[0].subplots(1, 2, sharey=True)\n", + " dat = np.abs(uvd.get_data(a,a))\n", + " #axsLeft.set_title('TempDisc', fontsize = 18)\n", + " for ax in axsLeft:\n", + " for j,p in enumerate(pols):\n", + " with warnings.catch_warnings():\n", + " warnings.simplefilter(\"ignore\")\n", + " for f,freq in enumerate(freqind):\n", + " ax.plot(dat[:,freq,j],label=np.around(freq_array[freq],1),color=cmap(color_ind[f]),linewidth=0.95)\n", + " if i==0 and j==0:\n", + " ax.legend()\n", + " ax.set_title(f'Pol {p}', fontsize = 16)\n", + " ax.set_xlabel('LST (hours)')\n", + " ax.set_ylabel('Power (A)')\n", + " xticks = [int(i) for i in np.linspace(0,len(lsts)-1, 5)]\n", + " ax.set_xticks(xticks)\n", + " ax.set_xticklabels(np.around(lsts[xticks], 2))\n", + " sm = cm.ScalarMappable(cmap=cmap)\n", + " #if freq==freqind[0]:\n", + " #cbar = plt.colorbar(sm)\n", + " #cbar.set_ticks(color_ind)\n", + " #cbar.set_ticklabels(freqs_use)\n", + " #ax.set_ylim('auto')\n", + " axsRight = subfigs[1].subplots(1, 1, sharex=True)\n", + " for f, freq in enumerate(freqind):\n", + " sm = cm.ScalarMappable(cmap=cmap)\n", + " if freq==freqind[0]:\n", + " axsRight.vlines(freqs_use, 0, 2.5e6,\n", + " color = cmap(color_ind),\n", + " linewidth = 2,\n", + " linestyle = '--')\n", + "# ax.plot(freq_array[0, :],dat[0, :])\n", + " status = get_ant_status(h, a)\n", + " axsRight = axes[i,j]\n", + " axsRight.set_xlim(xlim)\n", + " axsRight.set_ylim(ylim)\n", + " px, = axsRight.plot(freqs, 10*np.log10(np.abs(uvdx.get_data((a, a))[t_index])), color='r', alpha=0.75, linewidth=1)\n", + " py, = axsRight.plot(freqs, 10*np.log10(np.abs(uvdy.get_data((a, a))[t_index])), color='b', alpha=0.75, linewidth=1)\n", + " axsRight.grid(False, which='both')\n", + " abb = status_abbreviations[status]\n", + " axsRight.set_title(f'{a} ({abb})', fontsize=10, backgroundcolor=status_colors[status])\n", + " if k == 0:\n", + " axsRight.legend([px, py], ['NN', 'EE'])\n", + " if i == len(inclNodes)-1:\n", + " [t.set_fontsize(10) for t in axsRight.get_xticklabels()]\n", + " axsRight.set_xlabel('freq (MHz)', fontsize=10)\n", + " else:\n", + " axsRight.set_xticklabels([])\n", + " if j!=0:\n", + " axsRight.set_yticklabels([])\n", + " else:\n", + " [t.set_fontsize(10) for t in axsRight.get_yticklabels()]\n", + " axsRight.set_ylabel(r'$10\\cdot\\log$(amp)', fontsize=10)\n", + " j += 1\n", + " k += 1\n", + " for k in range(j,maxants):\n", + " axes[i,k].axis('off')\n", + " axes[i,maxants-1].annotate(f'Node {n}', (1.1,.3),xycoords='axes fraction',rotation=270) \n", + " \n", + " plt.suptitle(f'Antenna{a}, snap {snaploc}, input {snapinput}',fontsize=27)\n", + " plt.show()\n", + " plt.close() " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "plot_multi_freq_RD(uvd,uvdx,uvdy,freqind = [0,200,600,800,1000,1200,1400,1530],pols=['xx','yy'])" + ] + }, { "cell_type": "code", "execution_count": null, @@ -424,7 +687,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.4" + "version": "3.9.7" }, "toc": { "base_numbering": 1, From 35d8fd6b002d028799f6a05906c947fb53b1cc62 Mon Sep 17 00:00:00 2001 From: Ellie McLaurin Date: Tue, 4 Oct 2022 13:37:19 -0600 Subject: [PATCH 2/2] added plot_multi_freqs function to utils, to be used in nightly data_inspect notebooks --- hera_notebook_templates/utils.py | 185 ++++++++++++++++++++++++++++++- 1 file changed, 181 insertions(+), 4 deletions(-) diff --git a/hera_notebook_templates/utils.py b/hera_notebook_templates/utils.py index 28d2417..320eff0 100644 --- a/hera_notebook_templates/utils.py +++ b/hera_notebook_templates/utils.py @@ -18,7 +18,7 @@ import scipy from pyuvdata import UVCal, UVData, UVFlag, utils -from astropy.time import Time +from astropy.time import Time, TimeDelta from astropy.coordinates import EarthLocation, AltAz, Angle from astropy.coordinates import SkyCoord as sc from astropy.io import fits @@ -37,11 +37,10 @@ import matplotlib.gridspec as gridspec from matplotlib.ticker import FormatStrFormatter from matplotlib.lines import Line2D -from matplotlib import colors +from matplotlib import colors, cm import hera_qm -from hera_mc import cm_hookup, geo_sysdef -from hera_mc import cm_active +from hera_mc import cm_hookup, geo_sysdef, cm_active, mc import uvtools as uvt from uvtools import dspec @@ -2985,3 +2984,181 @@ def most_common_flag_rationale(self, jds=None): return sorted(mms.items(), key=lambda item: item[1])[-1][0] return np.nan + +def plot_multi_freqs(uvd,uvdx,uvdy,HHautos,JD,freqind = [0,200,400,600,800,1000,1200,1400,1530],pols=['NN','EE'], use_database = True, array_signal = 'antenna'): + """Plot power vs time for a collection of individual frequencies; also create an accompanying power vs frequency plot for a single time within the night. + + Parameters: + ---------- + uvd: UVData Object + Object containing autos from sum files. Used for getting antenna information. + uvdx: UVData Object + Sample observations from a single time during the desired night, on the xx (NN) polarization. + uvdy: + Sample observations from a single time during the desired night, on the yy (EE) polarization. + freqind: List + Frequency array index values. Allows for evenly spaced frequencies to be easily chosen and plotted. + Default values are [0,200,400,600,800,1000,1200,1400,1530], which correspond to freqeuncies [46.9, 71.3, 95.7, 120.2, 144.6, 169.0, + 193.4, 217.8, 233.7] MHz. + pols: List + Polarizations to plot. Can include any polarization strings accepted by pyuvdata. + use_database: Boolean + Option to attempt using the database to determine the array's commanded observing mode on the desired night. Default is True. + array_signal: String + Used to manually specify the expected array mode if use_database is set to False, or if database doesn't return an array mode. + Must be one of these values: 'antenna','noise','load','digital_noise_same','digital_noise_different'. Default is 'antenna'. + + Returns: + ------- + None + """ + + if use_database is True: + try: + db = mc.connect_to_mc_db(None) + session = db.sessionmaker() + startJD = float(HHautos[0].split('zen.')[1].split('.sum')[0]) + stopJD = float(HHautos[-1].split('zen.')[1].split('.sum')[0]) + start_time = Time(startJD,format='jd') + stop_time = Time(stopJD,format='jd') + # get initial state by looking for commands up to 3 hours before the starttime + # this logic can be improved after an upcoming hera_mc PR + # which will return the most recent command before a particular time. + search_start_time = start_time - TimeDelta(3*3600, format="sec") + initial_command_res = session.get_array_signal_source(starttime=search_start_time, stoptime=start_time) + if len(initial_command_res) == 0: + initial_source = "Unknown" + elif len(initial_command_res) == 1: + initial_source = initial_command_res[0].source + else: + # multiple commands + times = [] + sources = [] + for obj in initial_command_res: + times.append(obj.time) + sources.append(obj.source) + initial_source = sources[np.argmax(times)] + except Exception as e: + print(e) + initial_source = None + + if initial_source is None: + print(f"Database not found. Assuming array mode is '{array_signal}'.") + elif initial_source == 'Unknown': + print(f"Array mode not found. Assuming array mode is {array_signal}") + else: + print(f"Using database; array mode is '{initial_source}'. Some antennas may work in a different mode due to communication error.") + else: + if array_signal not in ['antenna','noise','load','digital_noise_same','digital_noise_different']: + raise ValueError("array_signal must be one of these values: 'antenna','noise','load','digital_noise_same','digital_noise_different'") + else: + print(f"use_database set to False. Assuming array mode is '{array_signal}'.") + + + plt.subplots_adjust(hspace=.0) + cmap = cm.get_cmap("plasma") + loc = EarthLocation.from_geocentric(*uvd.telescope_location, unit='m') + freq_array = uvd.freq_array[0]*1e-6 + freqs_use = freq_array[freqind] + times = uvdx.time_array + lsts = uvd.lst_array*3.819719 + inds = np.unique(lsts,return_index=True)[1] + lsts = np.asarray([lsts[ind] for ind in sorted(inds)]) + color_ind = np.linspace(0,1,len(freqind)) + + nodes, antDict, inclNodes = generate_nodeDict(uvdx) + ants = uvdx.get_ants() + sorted_ants = sort_antennas(uvdx) + #maxants = 0 + #for node in nodes: + #n = len(nodes[node]['ants']) + #if n>maxants: + #maxants = n + + #Nants = len(ants) + #Nside = maxants + #Yside = len(inclNodes) + + #t_index = 0 + #jd = times[t_index] + # utc = Time(jd, format='jd').datetime + + h = cm_active.get_active(at_date=JD, float_format="jd") + + state_plots = {'antenna': (0, 4e7, 'black', 1), 'noise' : (0, 4e6, 'cornflowerblue', 2), 'load' : (0,4e6,'green',2), 'digital_noise_same' : (1e7,4e7,'indigo',2), 'digital_noise_different': (1e7,4e7,'palevioletred',2)} + if use_database is True and (initial_source is not None and initial_source != "Unknown"): + mymin, mymax, my_color, my_linewidth = state_plots[initial_source] + else: + mymin, mymax, my_color, my_linewidth = state_plots[array_signal] + + for i,n in enumerate(inclNodes): + ants = nodes[n]['ants'] + j = 0 + k = 0 + for _,a in enumerate(sorted_ants): + if a not in ants: + continue + snaploc = antDict[a]['snapLocs'] + snapinput = antDict[a]['snapInput'] + fig = plt.figure(constrained_layout=True, figsize=(20, 4)) + subfigs = fig.subfigures(1, 2, wspace=0.07, width_ratios=[2, 1]) + axsLeft = subfigs[0].subplots(1, 2, sharey=True) + dat = np.abs(uvd.get_data(a,a)) + status = get_ant_status(h, a) + abb = status_abbreviations[status] + for j,p in enumerate(pols): + ax = axsLeft[j] + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + for f,freq in enumerate(freqind): + ax.plot(dat[:,freq,j],label=np.around(freq_array[freq],1),color=cmap(color_ind[f]),linewidth=0.95) + if a==0 and j==0: + ax.legend() + ax.set_title(f'{p}', fontsize = 14) + ax.set_xlabel('LST (hours)') + ax.set_ylabel('Power (amp)') + xticks = [int(i) for i in np.linspace(0,len(lsts)-1, 5)] + ax.set_xticks(xticks) + ax.set_xticklabels(np.around(lsts[xticks], 2)) + sm = cm.ScalarMappable(cmap=cmap) + if np.mean(dat)>mymin and np.mean(dat)