1GRDvs64SIR_HIST_SD-updates022222-MBwesternUS.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load in all the modules needed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "%pylab is deprecated, use %matplotlib inline and import the required libraries.\n",
      "Populating the interactive namespace from numpy and matplotlib\n"
     ]
    }
   ],
   "source": [
    "%pylab notebook\n",
    "import glob\n",
    "import matplotlib.pyplot as plt\n",
    "from mpl_toolkits.axes_grid1 import make_axes_locatable\n",
    "from mpl_toolkits.basemap import Basemap\n",
    "from netCDF4 import Dataset, num2date\n",
    "import numpy as np\n",
    "import os\n",
    "if os.name == 'nt':\n",
    "    os.environ[\"PROJ_LIB\"] = os.path.join(os.environ[\"CONDA_PREFIX\"], \"Library\", \"share\")\n",
    "    os.environ[\"GDAL_DATA\"] = os.path.join(os.environ[\"CONDA_PREFIX\"], \"Library\", \"share\", \"gdal\")\n",
    "import pandas as pd\n",
    "from pathlib import Path\n",
    "import re\n",
    "\n",
    "from cetbtools.ease2conv import Ease2Transform\n",
    "\n",
    "# the set_trace() command is helpful for entering interactive\n",
    "# debugging mode to step through lines of code in this notebook\n",
    "from IPython.core.debugger import set_trace"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set pandas display options to display entire DataFrame, use None for all \n",
    "pd.set_option('display.max_rows', 500)\n",
    "pd.set_option('display.min_rows', 120)\n",
    "pd.set_option('display.max_columns', 10)\n",
    "pd.set_option('display.width', 120)\n",
    "pd.set_option('display.max_colwidth', 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define the local machine location of CETB data cubes\n",
    "# This directory is expected to contain subdirectories in the following hierarchy\n",
    "# that duplicates the hierarchy on the Google Shared Drive NSIDC-SD-CETB/v1/, \n",
    "# for example:\n",
    "# dataDir/F13_SSMI/N/nc_cubes/cubes_<regionName>\n",
    "user = 'Joan' #Mariah #MJWindows\n",
    "if ('Joan' == user):\n",
    "    dataDir = '/mnt/data3/cetb/nsidc0630_v1/' #jmr machine fringe \n",
    "    scriptDir = '/mnt/data3/cetb/ipynb_melt_onset/scripts'\n",
    "    outDir = '/mnt/data3/cetb/ipynb_melt_onset_plots'\n",
    "    outNoisy = Path(Path.home(), 'noisy')\n",
    "elif ('Mariah' == user):\n",
    "    dataDir = Path(Path.home(), 'nsidc0630_v1') # Mariah's PC or Mary Jo's Mac\n",
    "    scriptDir = Path(Path.home(), 'ipynb_melt_onset', 'scripts')\n",
    "    outDir = Path(Path.home(), 'ipynb_melt_onset_plots') \n",
    "    outNoisy = Path(Path.home(), 'noisy')\n",
    "elif ('MJWindows' == user):\n",
    "    dataDir = Path('Z:/mj On My Mac/nsidc0630_v1') # Mary Jo's Windows machine\n",
    "    scriptDir = Path(Path.home(), 'ipynb_melt_onset', 'scripts')\n",
    "    outDir = Path(Path.home(), 'ipynb_melt_onset_plots')\n",
    "    outNoisy = Path(Path.home(), 'noisy')\n",
    "elif ('Mahboubeh' == user):\n",
    "    dataDir = Path('Z:/mj On My Mac/nsidc0630_v1') # Mary Jo's Windows machine\n",
    "    scriptDir = Path(Path.home(), 'ipynb_melt_onset', 'scripts')\n",
    "    outDir = Path(Path.home(), 'ipynb_melt_onset_plots')\n",
    "    outNoisy = Path(Path.home(), 'noisy')    \n",
    "else:\n",
    "    raise ValueError(\"unknown user= %s\\n\" % (user) )\n",
    "    \n",
    "# Make the output directory for plot images, if it doesn't yet exist\n",
    "if not os.path.exists(outDir):\n",
    "    os.makedirs(outDir)\n",
    "    \n",
    "if not os.path.exists(outNoisy):\n",
    "    os.makedirs(outNoisy)\n",
    "    \n",
    "noisydirs = ['sd', 'noisytb_csv', 'noisydav_csv', 'noisytbSIR_csv', 'noisydavSIR_csv']\n",
    "for dir in noisydirs:\n",
    "    nextDir = Path(outNoisy, dir)\n",
    "    if not os.path.exists(nextDir):\n",
    "        os.makedirs(nextDir)\n",
    "    \n",
    "%cd $scriptDir\n",
    "dataDir, user"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "noisydirs = ['sd', 'noisytb_csv', 'noisydav_csv', 'noisytbSIR_csv', 'noitydavSIR_csv']\n",
    "for dir in noisydirs:\n",
    "    print('dir=%s' % dir)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from CETB_IO import read_Tb\n",
    "from CETB_IO import read_Tb_std_dev\n",
    "from CETB_IO import coords\n",
    "from CETB_IO import read_Tb_whole\n",
    "from CETB_IO import find_cube_offset\n",
    "from CETB_IO import grid_locations_of_subset\n",
    "from CETB_IO import years_for\n",
    "from CETB_algorithms import DAV_MOD\n",
    "from CETB_algorithms import XPGR\n",
    "from CETB_algorithms import calc_DAV\n",
    "from CETB_algorithms import end_high_DAV\n",
    "from CETB_algorithms import D_DAV\n",
    "from CETB_algorithms import Winter_DAV\n",
    "from CETB_analysis import Tb_hist_annual\n",
    "from CETB_analysis import DAV_hist_annual\n",
    "from CETB_analysis import Tb_hist_monthly\n",
    "from CETB_analysis import DAV_hist_monthly\n",
    "from CETB_analysis import TbDAV_series_one_year\n",
    "from CETB_analysis import early_melt_events\n",
    "from CETB_analysis import min_max_series\n",
    "from CETB_analysis import MOD_array"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Specify inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Specify region, satellite, sensor, channel, and image reconstruction algorithm of interest in file name\n",
    "# this notebook will read in 2 CETB datasets so that channels/algorithms/sensors can be compared\n",
    "region='WesternUS'  #make this the same syntax as cubefilenames and sub-directory #GLaIL #AKYukon\n",
    "sat_GRD='F13'   #'AQUA' for AMSRE, 'F13','F14','F15'... for SSMI\n",
    "sat_SIR= 'F13'\n",
    "sensor_GRD='SSMI'  #'AMSRE', 'SSMI', etc.\n",
    "sensor_SIR='SSMI'\n",
    "channel_GRD='37V'  #'36V','36H', '18V','18H', etc. '19V','19H' and '37V','37H' for SSMI)\n",
    "channel_SIR='37V'\n",
    "alg_GRD='GRD'   #SIR or GRD\n",
    "alg_SIR='SIR'\n",
    "\n",
    "\n",
    "# set the sir to grd factor, depends on the channel\n",
    "if (re.match('^[389]', channel_GRD)):\n",
    "    sir_2_grd_factor = 8 # assume 3.125 km to 25 km\n",
    "elif (re.match('^[12]', channel_GRD)):\n",
    "    sir_2_grd_factor = 4 # assume 6.25 km to 25 km\n",
    "else:\n",
    "    raise ValueError(\"Cannot determine sir_2_grd_factor from channel %s\\n\" % (channel_GRD) )\n",
    "    \n",
    "cubeType_GRD = channel_GRD + '-' + alg_GRD\n",
    "cubeType_SIR = channel_SIR + '-' + alg_SIR\n",
    "  \n",
    "if ('SSMI' == sensor_GRD) or ('SSMIS' == sensor_GRD):\n",
    "    provider='CSU' \n",
    "    version='v1.*'\n",
    "elif 'AMSRE' == sensor_GRD:\n",
    "    provider='RSS'\n",
    "    version='v1.3'\n",
    "\n",
    "\n",
    "Years_all=years_for(sat_GRD)\n",
    "#might want to truncate Years to subset if very slow during testing\n",
    "#if we give it more years than available what do we want it to do? \n",
    "#warn me but return what it finds\n",
    "    \n",
    "hemName = 'N'    \n",
    "    \n",
    "# either change to the directory where the data is or specify it\n",
    "datadir_GRD = \"%s/%s_%s/%s/nc_cubes/cubes_%s\" % (\n",
    "    dataDir, sat_GRD, sensor_GRD, hemName, region)\n",
    "datadir_SIR = \"%s/%s_%s/%s/nc_cubes/cubes_%s\" % (\n",
    "    dataDir, sat_SIR, sensor_SIR, hemName, region)\n",
    "\n",
    "\n",
    "# filepath patterns for each type of cubefile\n",
    "prefix_GRD = \"CETB.cubefile.%s.%s_%s-%s-%s-%s-%s\" % (\n",
    "    region, sat_GRD, sensor_GRD, channel_GRD, alg_GRD, provider, version)\n",
    "prefix_SIR = \"CETB.cubefile.%s.%s_%s-%s-%s-%s-%s\" % (\n",
    "    region, sat_SIR, sensor_SIR, channel_SIR, alg_SIR, provider, version)\n",
    "\n",
    "# on MJ's machine\n",
    "hemName = 'N'\n",
    "\n",
    "#Enter a site name for titles of plots\n",
    "# enter latitutde and longitude in decimal degrees\n",
    "\n",
    "#Site='Komsomolets Is., Severnaya Zemlya AWS, Russia'\n",
    "#lat_start=80.51666667   \n",
    "#lon_start=94.81666667\n",
    "\n",
    "Site='Fraser, Fool Creek (FF)'\n",
    "lat_start=39.8810   \n",
    "lon_start=-105.8679\n",
    "\n",
    "#Site='Low Relief Test, Alberta 49.73,-111.14'\n",
    "\n",
    "#Site='Great Lakes Test Site, Lake Superior E of Duluth'\n",
    "#lat_start=47.353097   \n",
    "#lon_start=-89.861310\n",
    "\n",
    "#AKYukon Sites\n",
    "#Site='Barrow SE, AK'# 3 km SE of the airport pixel, GRD and SIRs should not be coastal \n",
    "    #Note is within the same GRD as the airport\n",
    "#lat_start=71.2709\n",
    "#lon_start=-156.694\n",
    "\n",
    "#Site='Barrow airport, AK'\n",
    "#lat_start=71.28181\n",
    "#lon_start=-156.772\n",
    "\n",
    "#Site='45 km S of Barrow airport, AK'\n",
    "#lat_start=70.908886\n",
    "#lon_start=-156.56875\n",
    "\n",
    "#Site='114 km South of Barrow airport, AK' #114 km south of airport to get an inland site\n",
    "#lat_start=70.28181a\n",
    "#lon_start=-156.772\n",
    "\n",
    "#Site ='Upper Kuparuk Basin, AK' #selected by VJ\n",
    "#lat_start=68.62421667\n",
    "#lon_start=-149.5250722\n",
    "\n",
    "#Site='Teshekpuk Lake 1'#near Barrow, AK (Permafrost installation NW margin of lake)'\n",
    "#lat_start=70.722902\n",
    "#lon_start=-153.836329\n",
    "\n",
    "#Site='Teshekpuk Lake 2; center' #, near Barrow, AK (middle of lake)'\n",
    "#lat_start=70.58333675326588\n",
    "#lon_start=-153.452493001114\n",
    "\n",
    "#Western_CA Sites\n",
    "#Site='NWT C57 04242006 Spring Migration'\n",
    "#lat_start=62.61\n",
    "#lon_start=-109.64\n",
    "\n",
    "#Site='Great Slave Lake'\n",
    "#lat_start=61.87167\n",
    "#lon_start=-114.05\n",
    "\n",
    "#GLAIL Sites\n",
    "#Site='Iceland, Central Vatnajokull'\n",
    "#lat_start=64.442   \n",
    "#lon_start=-16.730 \n",
    "\n",
    "#Site='Iceland, Katla Caldera Test'\n",
    "#lat_start=63.64361111   \n",
    "#lon_start=-19.20138889\n",
    "\n",
    "#Site='Iceland, Vatnajokull Site 1'\n",
    "#lat_start=64.64903   \n",
    "#lon_start=-17.32128\n",
    "\n",
    "#Site = 'Iceland, Vatnajokull Site 2'\n",
    "#lat_start=64.25860278\n",
    "#lon_start=-16.93241667\n",
    "\n",
    "#Site='Iceland, Vatnajokull Site 3'\n",
    "#lat_start=64.423333\n",
    "#lon_start=-16.77305556\n",
    "#Site='Iceland, Vatnajokull Site 4'\n",
    "#lat_start=64.66222\n",
    "#lon_start=-17.399444\n",
    "\n",
    "#Western_US Sites\n",
    "#Site='Senator Beck Basin (SASP), Colorado'\n",
    "#lat_start=37.9069   \n",
    "#lon_start=-107.710\n",
    "\n",
    "if Site=='Rabbit Ears':\n",
    "    lat_start=40.5  #southern boundary\n",
    "    lat_end=40.5  #northern boundary\n",
    "    lon_start=-106.7  #western boundary\n",
    "    lon_end=-106.7 #eastern boundary\n",
    "elif Site=='Fraser':\n",
    "    lat_start=39.85  #southern boundary\n",
    "    lat_end=40.0  #northern boundary\n",
    "    lon_start=-106.0  #western boundary\n",
    "    lon_end=-105.9 #eastern boundary\n",
    "elif Site=='North Park':\n",
    "    lat_start=40.7  #southern boundary\n",
    "    lat_end=40.7  #northern boundary\n",
    "    lon_start=-106.15  #western boundary\n",
    "    lon_end=-106.15 #eastern boundary\n",
    "elif Site=='CLPX-LSA':\n",
    "    lat_start=39.7  #southern boundary\n",
    "    lat_end=41.0  #northern boundary\n",
    "    lon_start=-106.7  #western boundary\n",
    "    lon_end=-105.4 #eastern boundary\n",
    "elif Site=='CUSTOM':\n",
    "    lat_start=43  #southern boundary\n",
    "    lat_end=43  #northern boundary\n",
    "    lon_start=-110  #western boundary\n",
    "    lon_end=-110 #eastern boundary\n",
    "elif Site=='Senator Beck':\n",
    "    # enter latitutde and longitude in decimal degrees\n",
    "    lat_start=37.9069   #southern boundary\n",
    "    lat_end=37.9069   #northern boundary\n",
    "    lon_start=-107.726   #western boundary\n",
    "    lon_end=-107.726 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# check that the GRD filename prefix is really what we have on the local machine\n",
    "prefix_GRD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get cube offset for finding row/col\n",
    "find_cube_offset(region, cubeDir=datadir_SIR, cubeType=cubeType_SIR, verbose=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# row and col numbers of the GRD pixels that include this region\n",
    "rows_cols = grid_locations_of_subset(region, lat_start, lon_start, cubeDir=datadir_GRD)\n",
    "rows_cols_GRD=[rows_cols[0][2],rows_cols[0][2]+1, rows_cols[1][2], rows_cols[1][2]+1]\n",
    "rows_cols_GRD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# row and col numbers of the envelope of 64 SIR pixels\n",
    "#MJB: I am not convinced that this is a good approach to calculate these coordinates\n",
    "#I need to look at this a little more closely\n",
    "rows_cols_env=[rows_cols_GRD[0]*sir_2_grd_factor,rows_cols_GRD[1]*sir_2_grd_factor, rows_cols_GRD[2]*sir_2_grd_factor, rows_cols_GRD[3]*sir_2_grd_factor]\n",
    "rows_cols_env"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#build the string that is the index to the SIR cell closest to our site\n",
    "Site_nearest_row_col = \"%s,%s\" % (str(rows_cols[0][0]),str(rows_cols[1][0])) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# check that the directory, filename pattern and Years are what we expect\n",
    "# subset Years to a shorter set if only testing, because this takes a long time\n",
    "subYears = Years_all[7:10]\n",
    "datadir_SIR, prefix_SIR, subYears"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#new cell to read whole dataset\n",
    "#takes a long time, for testing substitute a year for Years\n",
    "#subYears = Years[0:2]\n",
    "\n",
    "data_SIR = read_Tb_whole(datadir_SIR, prefix_SIR, subYears,\n",
    "                         rows_cols_env[0], rows_cols_env[1], rows_cols_env[2], rows_cols_env[3])\n",
    "\n",
    "#FIXME: instead of copying these elements into new variables, just\n",
    "#use the values directly in subsequent cells.  This is a more direct way to\n",
    "#work with the output from the read_Tb_whole function.  It also would\n",
    "#make it clear in subsequent cells that, for example \"cal_month\" is the\n",
    "#month output from the SIR call.\n",
    "#CETB_SIR = data_SIR['TB']   # 3-D Tb time-series array\n",
    "##cal_date = data_SIR['cal_date']    # 1-D array of dates\n",
    "#cal_year = data_SIR['cal_year']    # 1-D array of years\n",
    "#cal_month = data_SIR['cal_month']   # 1-D array of months\n",
    "\n",
    "#FIXME: same issue here, instead of copying the GRD TB data, subsequent\n",
    "#cells should just use data_GRD['TB']\n",
    "data_GRD = read_Tb_whole(datadir_GRD, prefix_GRD, subYears, rows_cols_GRD[0], rows_cols_GRD[1], rows_cols_GRD[2], rows_cols_GRD[3])\n",
    "CETB_GRD = data_GRD['TB']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#region version, may not need\n",
    "# 'data' will be a tuple with the first element as the Tb time-series data, days, months, and years \n",
    "# to be used in plot/algorithm functions later\n",
    "#data_SIR=read_Tb(datadir_SIR, prefix_SIR, Years,rows_cols_env[0],rows_cols_env[1],rows_cols_env[2],rows_cols_env[3])\n",
    "#CETB_SIR=data_SIR[0]   # 3-D Tb time-series array\n",
    "#cal_date=data_SIR[1]    # 1-D array of dates\n",
    "#cal_year=data_SIR[2]    # 1-D array of years\n",
    "#cal_month=data_SIR[3]   # 1-D array of months\n",
    "#data_GRD=read_Tb(datadir_GRD, prefix_GRD, Years,rows_cols_GRD[0],rows_cols_GRD[1],rows_cols_GRD[2],rows_cols_GRD[3])\n",
    "#CETB_GRD=data_GRD[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# calculate DAV for the Tb data that was imported\n",
    "DAV_GRD = calc_DAV(data_GRD['TB'])\n",
    "DAV_SIR = calc_DAV(data_SIR['TB'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# calculate melt onset date, each row returned is the MOD for that year, \n",
    "# shows '3' with the first pixel in the subset to melt\n",
    "# can tweak thresholds and count/window, legacy algorithm used \n",
    "# 3 occurrences in 5 day window"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#MOD_SIR1, melt_flag_df = DAV_MOD(DAV_threshold, Tb_threshold, occurrences, window, \n",
    "#                                 DAV_SIR, CETB_SIR, Years, cal_year, cal_date, \n",
    "#                                 rows_cols_env)\n",
    "#MOD_SIR1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# New output from MOD_array\n",
    "\n",
    "MOD_DOY_array: this is a masked array (used to be the only thing Mitch returned), contains doy of average melt over all years for each pixel\n",
    "\n",
    "MOD_DOY_df: MJB: I think this is more useful, it is a data frame with 1 row for each pixel, and 1 column for each year, plus an extra column for the 'Avg' Contents of the data are doy of first time melt criteria were met. 'Avg' column is the contents of MOD_DOY_array\n",
    "\n",
    "meltflag_df: This may also be useful for further analysis, 1 row for each date (2 dates per day), 1 column for each pixel. Values are 1 if DAV/TB thresholds were exceeded on this date, 0 otherwise."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#set up MOD parameters\n",
    "# can tweak thresholds and count/window, legacy algorithm used \n",
    "# 3 occurrences in 5 day window and shows '3' (count) with the first pixel in the subset to melt because of the counter\n",
    "#3 out of5 days is common for lower latitudes or intermittent melt (But not too intermittent)\n",
    "window = 1  #for 5 days change to 10\n",
    "count = 1   #for 3 occurrences change to 3\n",
    "if ('SSMI' == sensor_GRD) or ('SSMIS' == sensor_GRD):\n",
    "    DAV_threshold=10 \n",
    "    Tb_threshold=247\n",
    "elif 'AMSRE' == sensor_GRD:\n",
    "    DAV_threshold=18 \n",
    "    Tb_threshold=252"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get array of MODs for each pixel in the 64 set to use for map\n",
    "# This will contain each year and the Avg MOD for all years\n",
    "# calculate melt onset date, each row returned is the MOD for that year, \n",
    "MOD_DOY_array, MOD_DOY_df, meltflag_df = MOD_array(\n",
    "    datadir_SIR, prefix_SIR, data_SIR, DAV_SIR, rows_cols_env, \n",
    "    subYears, window, count, DAV_threshold, Tb_threshold)\n",
    "MOD_DOY_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# MOD of the GRD pixel \n",
    "# Use the same thresholds/criteria as for SIR\n",
    "MOD_DOY_array_GRD, MOD_DOY_GRD_df, meltflag_GRD_df = MOD_array(\n",
    "    datadir_GRD, prefix_GRD, data_GRD, DAV_GRD, rows_cols_GRD, \n",
    "    subYears, window, count, DAV_threshold, Tb_threshold)\n",
    "MOD_DOY_GRD_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Trial-and-error decide on size for GRD vs. SIR scatter markers\n",
    "#np.sqrt((1200)/8)^2\n",
    "sSIR = 100\n",
    "sGRD = 3200"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "outFile = Path(outDir, 'SevZ_AWS_F13_1999-2000_MOD_win1d_1GRD.png')\n",
    "\n",
    "# PLOT the MOD MAP - avg of all years or one year of interest\n",
    "# Decide here:\n",
    "#plot_type='all' # DO THIS to show avg of all years\n",
    "# OR\n",
    "plot_type='year' # DO THIS and choose a year for 1 year of interest\n",
    "year=2000\n",
    "## fixme ## the scale bar does not work for single year (values range -.1 -+.1)\n",
    "\n",
    "if plot_type == 'all':\n",
    "    SIR_array = MOD_DOY_df['Avg'].values\n",
    "    GRD_array = MOD_DOY_GRD_df['Avg'].values\n",
    "    label = 'Avg DOY ' + str(Years[0]) + '-' + str(Years[-1])\n",
    "    title = sensor_SIR + '-' + Site + ' - MOD - (' + label + ')'\n",
    "elif plot_type == 'year':\n",
    "    SIR_array = MOD_DOY_df[year].values\n",
    "    GRD_array = MOD_DOY_GRD_df[year].values\n",
    "    label = str(year) + 'DOY' \n",
    "    title = sensor_SIR + '-' + Site + ' - MOD - (' + label + ')'\n",
    "\n",
    "# Set a few common things\n",
    "#min day of year to plot--look at both data arrays and subtract 2 days\n",
    "minday = np.amin([np.amin(GRD_array), np.amin(SIR_array)]) - 2\n",
    "#max day of year to plot same here but add 2 days\n",
    "maxday = np.amax([np.amax(GRD_array), np.amax(SIR_array)]) + 2\n",
    "\n",
    "graticule_fontsize = 4\n",
    "\n",
    "fig, axes = plt.subplots(1, 2, figsize=(10,8))\n",
    "\n",
    "axes[0].set_title(title)\n",
    "# create polar stereographic Basemap instance.\n",
    "m = Basemap(ax=axes[0], projection='stere',lon_0=lon_start,lat_0=lat_start,lat_ts=90.,\\\n",
    "            llcrnrlat=(lat_start-.7),urcrnrlat=(lat_start+.7),\\\n",
    "            llcrnrlon=(lon_start-.7),urcrnrlon=(lon_start+.7),\\\n",
    "            rsphere=6371200.,resolution='l',area_thresh=10000, epsg=3857)\n",
    "m.drawcoastlines()\n",
    "m.drawcountries()\n",
    "m.drawstates()\n",
    "#m.etopo(scale=5, alpha=0.7)\n",
    "#m.bluemarble(scale=1)\n",
    "parallels = np.arange(0.,90,0.5)\n",
    "m.drawparallels(parallels,labels=[1,0,0,0],fontsize=10)\n",
    "# draw meridians\n",
    "meridians = np.arange(180.,360.,0.5)\n",
    "m.drawmeridians(meridians,labels=[0,0,0,1],fontsize=10)\n",
    "m.arcgisimage(service='World_Physical_Map', xpixels = 1500, verbose= True)\n",
    "\n",
    "x,y = m(pixel_lons, pixel_lats)\n",
    "#m.plot(x,y, 'bs', markersize=11, markeredgewidth=1, alpha=0.2)\n",
    "im0 = m.scatter(x, y, c=SIR_array, s=sSIR, marker='s', lw=0, cmap='BuPu_r', alpha=.8,\n",
    "          vmin=minday, vmax=maxday)\n",
    "\n",
    "# Get the current axes and configure placement so colorbar will\n",
    "# be 5% of ax width and the padding will be 0.08 in.\n",
    "divider = make_axes_locatable(axes[0])\n",
    "cax = divider.append_axes(\"right\", size=\"5%\", pad=0.08)\n",
    "cbar = plt.colorbar(im0, cax=cax, label='DOY')\n",
    "cbar.set_clim(minday, maxday)  #color bar limits\n",
    "\n",
    "# plot a red line on colorbar for the MOD for the GRD pixel\n",
    "cbar.ax.hlines(cbar.norm(GRD_array[0]), 0, 1, color='green') \n",
    "\n",
    "# create polar stereographic Basemap instance.\n",
    "m = Basemap(ax=axes[1], projection='stere',lon_0=lon_start,lat_0=lat_start,lat_ts=90.,\\\n",
    "            llcrnrlat=(lat_start-.7),urcrnrlat=(lat_start+.7),\\\n",
    "            llcrnrlon=(lon_start-.7),urcrnrlon=(lon_start+.7),\\\n",
    "            rsphere=6371200.,resolution='l',area_thresh=10000, epsg=3857)\n",
    "m.drawcoastlines()\n",
    "m.drawcountries()\n",
    "m.drawstates()\n",
    "#m.etopo(scale=5, alpha=0.7)\n",
    "#m.bluemarble(scale=1)\n",
    "parallels = np.arange(0.,90,0.5)\n",
    "m.drawparallels(parallels,labels=[1,0,0,0],fontsize=10)\n",
    "# draw meridians\n",
    "meridians = np.arange(180.,360.,0.5)\n",
    "m.drawmeridians(meridians,labels=[0,0,0,1],fontsize=10)\n",
    "m.arcgisimage(service='World_Physical_Map', xpixels = 1500, verbose= True)\n",
    "\n",
    "x,y = m(pixel_lons_GRD, pixel_lats_GRD)\n",
    "#m.plot(x,y, 'bs', markersize=11, markeredgewidth=1, alpha=0.2)\n",
    "im1 = m.scatter(x, y, c=GRD_array, s=sGRD, marker='s', lw=0, cmap='BuPu_r', alpha=.8,\n",
    "          vmin=minday, vmax=maxday)          \n",
    "\n",
    "# Get the current axes and configure placement so colorbar will\n",
    "# be 5% of ax width and the padding will be 0.05 in.\n",
    "divider = make_axes_locatable(axes[1])\n",
    "cax = divider.append_axes(\"right\", size=\"5%\", pad=0.05)\n",
    "cbar = plt.colorbar(im1, cax=cax, label='DOY')\n",
    "cbar.set_clim(minday, maxday)  #color bar limits\n",
    "\n",
    "plt.tight_layout(pad=3.0)\n",
    "plt.show()\n",
    "plt.savefig(outFile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load SIR Tb standard deviation data from netCDF file with function\n",
    "Tb_std_dev=read_Tb_std_dev(datadir_SIR, prefix_SIR, subYears,\n",
    "                           rows_cols_env[0],rows_cols_env[1],rows_cols_env[2],rows_cols_env[3])\n",
    "#Tb_std_dev"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Tb_std_dev.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_SIR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#need to run this cell for the max DAV to work\n",
    "y_dims_list=list(range(len(data_SIR['TB'][0,:,0])))  # creates a list of the y-dimension pixel indices, used for plotting\n",
    "x_dims_list=list(range(len(data_SIR['TB'][0,0,:])))\t# creates a list of the x-dimension pixel indices, for plotting\n",
    "y_s=list(range(rows_cols_env[0],rows_cols_env[1]))  # makes a list of the y(row) numbers so the dataframe of early melt events can be indexed\n",
    "x_s=list(range(rows_cols_env[2],rows_cols_env[3]))  # makes a list of the x(col) numbers so the dataframe of early melt events can be indexed\n",
    "\n",
    "# create dataframe of the 64 SIR pixels\n",
    "Tb_std_frame=pd.DataFrame()\n",
    "for i in y_dims_list:\n",
    "    for j in x_dims_list:\n",
    "        column=pd.DataFrame(data=Tb_std_dev[:,i,j], columns=[str(y_s[i])+','+str(x_s[j])])\n",
    "        Tb_std_frame=pd.concat([Tb_std_frame,column],axis=1)\n",
    "Tb_std_frame.index=data_SIR['cal_date']\n",
    "Tb_std_frame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load dataframe of Tb GRD for plotting Tb on specified day and observation\n",
    "Tb_std_dev_GRD=read_Tb_std_dev(datadir_GRD, prefix_GRD, subYears,\n",
    "                               rows_cols_GRD[0],rows_cols_GRD[1],rows_cols_GRD[2],rows_cols_GRD[3])\n",
    "Tb_std_dev_GRD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# slice of dataframe described above\n",
    "Tb_sd_frame_GRD=pd.DataFrame(Tb_std_dev_GRD[:,0,0], index=data_GRD['cal_date'])\n",
    "Tb_sd_frame_GRD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load dataframe of Tb GRD for plotting Tb on specified day and observation\n",
    "Tb_frame_GRD=pd.DataFrame(data_GRD['TB'][:,0,0], index=data_GRD['cal_date'])\n",
    "Tb_frame_GRD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load dataframe of Tb SIR for plotting Tb on specified day and observation\n",
    "Tb_frame=pd.DataFrame()\n",
    "for i in y_dims_list:\n",
    "    for j in x_dims_list:\n",
    "        column=pd.DataFrame(data=data_SIR['TB'][:,i,j], columns=[str(y_s[i])+','+str(x_s[j])])\n",
    "        Tb_frame=pd.concat([Tb_frame,column],axis=1)\n",
    "Tb_frame.index=data_SIR['cal_date']\n",
    "Tb_frame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get the min and max Tb of the 64 for each time step\n",
    "Tb_64_min=Tb_frame.min(axis=1)\n",
    "Tb_64_max=Tb_frame.max(axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# compute std dev of the 64 SIR pixels\n",
    "Tb_std_64=Tb_frame.std(axis=1)\n",
    "Tb_std_64"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# take difference of the GRD_std_dev variable and the computed sd of the 64 SIR pixels\n",
    "Tb_sd_diff=Tb_sd_frame_GRD[0]-Tb_std_64\n",
    "Tb_sd_diff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "DAV_GRD_frame=pd.DataFrame(DAV_GRD[:,0,0], index=data_GRD['cal_date'])\n",
    "DAV_GRD_frame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "DAV_SIR_frame=pd.DataFrame()\n",
    "for i in y_dims_list:\n",
    "    for j in x_dims_list:\n",
    "        column=pd.DataFrame(data=DAV_SIR[:,i,j], columns=[str(y_s[i])+','+str(x_s[j])])\n",
    "        DAV_SIR_frame=pd.concat([DAV_SIR_frame,column],axis=1)\n",
    "DAV_SIR_frame['Date']=data_SIR['cal_date']        \n",
    "DAV_SIR_frame=DAV_SIR_frame.set_index('Date')\n",
    "DAV_SIR_frame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Tb_nearest=DAV_SIR_frame[Site_nearest_row_col]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Site_nearest_row_col"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "outFile = Path(outDir, 'FF2002-2004_Timeseries.png')\n",
    "\n",
    "#Generate a time series of specified length (any window in dataset)\n",
    "#Make sure to update filename in last line to export new file\n",
    "fig,(ax1,ax2,ax3)=plt.subplots(3,1, sharex=True, figsize=(10,10))\n",
    "\n",
    "ax1.set_title(str(lat_start)+'_'+str(lon_start)+' '+Site)\n",
    "#Tb_sd_frame_GRD.plot(xlim=['2019-01-01','2019-12-31'])\n",
    "\n",
    "ax1.plot(Tb_64_max, color='red', label='SIR-Tb-max')\n",
    "ax1.plot(Tb_64_min, color='blue', label='SIR-Tb-min')\n",
    "#ax1.plot(Tb_nearest, color = 'black',label='SIR-Site')\n",
    "ax1.plot(Tb_frame_GRD, color='#636363', label='GRD-Tb')\n",
    "ax1.set_ylabel('Tb (K)', fontsize=16)\n",
    "ax1.set_ylim(150,300)\n",
    "#ax1.yticks(fontsize=12)\n",
    "ax1.legend(fontsize=12)\n",
    "ax2.plot(Tb_sd_frame_GRD, color='black', label='GRD-StDev')\n",
    "ax2.plot(Tb_std_frame[str(rows_cols[0][0])+','+str(rows_cols[1][0])], color='#fdbb84', label='SIR-1px')\n",
    "ax2.set_ylabel('Tb-STD (K)',fontsize=16)\n",
    "ax2.plot(Tb_std_64, color='#4292c6',label='SIR-64px-StDev')\n",
    "ax2.plot(Tb_sd_diff, label='StDevDiff_GRD-64SIR', color='green')\n",
    "ax2.legend(fontsize=10,loc='lower right')\n",
    "ax2.set_ylim(-12,30)\n",
    "ax3.plot(DAV_GRD_frame, color='black', label='GRD-DAV')\n",
    "ax3.plot(DAV_SIR_frame[str(rows_cols[0][0])+','+str(rows_cols[1][0])], color='#4292c6', label='SIR-DAV')\n",
    "ax3.set_ylabel('DAV (K)',fontsize=16)\n",
    "ax3.legend(fontsize=12)\n",
    "ax3.set_ylim(0,50)\n",
    "#ax3.set_xlim('2010-03-01', '2010-12-31') #\n",
    "#ax1.set_xlim('2010-01-01', '2011-12-31') #there is something wrong with this line (new since 10-26-21) \n",
    "#fix above line probably changing date format\n",
    "#related to matplotlib version maybe\n",
    "fig.autofmt_xdate()\n",
    "plt.savefig(outFile)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Tb_sd_diff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "outFile = Path(outDir, 'FF_SDdif2.png')\n",
    "\n",
    "# histogram of the difference between GRD_Tb_std variable and the computed std of the 64 SIR pixels\n",
    "#plt.close()\n",
    "fig,ax=plt.subplots()\n",
    "values=Tb_sd_diff.dropna().values\n",
    "ax.hist(values,bins=500, label='STD_Dif GRD-64SIR')\n",
    "ax.set_title(Site+' all data')\n",
    "ax.axvline(x=Tb_sd_diff.mean(), color='red', linestyle='dashed', label=Tb_sd_diff.mean().round(3))\n",
    "ax.legend(fontsize=12)\n",
    "ax.set_xlim(-10,+10) #adding in specified limits JRM in progress\n",
    "ax.set_ylabel('Frequency',fontsize=14)\n",
    "ax.set_xlabel('Standard Deviation Difference (K)',fontsize=14)\n",
    "plt.savefig(outFile)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "# PLOT the std dev of Tb MAP - \n",
    "date='2003-04-16' # specify date of interest\n",
    "obs=0 # 0 or 1 for morning or evening observation\n",
    "\n",
    "# load Tb_std_dev array\n",
    "array=Tb_std_frame.loc[date].values\n",
    "array=array[obs]\n",
    "GRD=Tb_sd_frame_GRD.loc[date].values\n",
    "GRD=GRD[obs]\n",
    "\n",
    "fig = plt.figure(figsize=(8,8))\n",
    "# create figure and axes instances\n",
    "#fig = plt.figure()\n",
    "ax = fig.add_axes([0.1,0.1,0.8,0.8])\n",
    "#ax = fig.add_subplot(121)\n",
    "# create polar stereographic Basemap instance.\n",
    "m = Basemap(projection='stere',lon_0=lon_start,lat_0=lat_start,lat_ts=90.,\\\n",
    "            llcrnrlat=(lat_start-.7),urcrnrlat=(lat_start+.7),\\\n",
    "            llcrnrlon=(lon_start-.7),urcrnrlon=(lon_start+.7),\\\n",
    "            rsphere=6371200.,resolution='l',area_thresh=10000, epsg=3857)\n",
    "m.drawcoastlines()\n",
    "m.drawcountries()\n",
    "m.drawstates()\n",
    "#m.etopo(scale=5, alpha=0.7)\n",
    "#m.bluemarble(scale=1)\n",
    "parallels = np.arange(0.,90,0.5)\n",
    "m.drawparallels(parallels,labels=[1,0,0,0],fontsize=10)\n",
    "# draw meridians\n",
    "meridians = np.arange(180.,360.,0.5)\n",
    "m.drawmeridians(meridians,labels=[0,0,0,1],fontsize=10)\n",
    "m.arcgisimage(service='World_Physical_Map', xpixels = 1500, verbose= True)\n",
    "\n",
    "x,y=m(pixel_lons, pixel_lats)\n",
    "#m.plot(x,y, 'bs', markersize=11, markeredgewidth=1, alpha=0.2)\n",
    "m.scatter(x,y,c=array, s=100, marker='s',lw=0, cmap='Oranges', alpha=.8)  # cmap='Blues'\n",
    "x,y=m(lon_start,lat_start)\n",
    "m.plot(x,y,'ro',markersize=4,markeredgewidth=0.0)\n",
    "plt.title('Tb - std dev - '+date)\n",
    "\n",
    "# plot a red line on colorbar for the MOD for the GRD pixel\n",
    "cbar=plt.colorbar()\n",
    "cbar.ax.hlines(cbar.norm(GRD), 0, 1, color='green') \n",
    "#cbar.ax.clim(0,1)  #color bar limits\n",
    "\n",
    "plt.show()\n",
    "\n",
    "fig = plt.figure(figsize=(8,8))\n",
    "ax = fig.add_axes([0.1,0.1,0.8,0.8])\n",
    "\n",
    "# create polar stereographic Basemap instance.\n",
    "m = Basemap(projection='stere',lon_0=lon_start,lat_0=lat_start,lat_ts=90.,\\\n",
    "            llcrnrlat=(lat_start-.7),urcrnrlat=(lat_start+.7),\\\n",
    "            llcrnrlon=(lon_start-.7),urcrnrlon=(lon_start+.7),\\\n",
    "            rsphere=6371200.,resolution='l',area_thresh=10000, epsg=3857)\n",
    "m.drawcoastlines()\n",
    "m.drawcountries()\n",
    "m.drawstates()\n",
    "#m.etopo(scale=5, alpha=0.7)\n",
    "#m.bluemarble(scale=1)\n",
    "parallels = np.arange(0.,90,0.5)\n",
    "m.drawparallels(parallels,labels=[1,0,0,0],fontsize=10)\n",
    "# draw meridians\n",
    "meridians = np.arange(180.,360.,0.5)\n",
    "m.drawmeridians(meridians,labels=[0,0,0,1],fontsize=10)\n",
    "m.arcgisimage(service='World_Physical_Map', xpixels = 1500, verbose= True)\n",
    "\n",
    "#x,y=m(pixel_lons, pixel_lats)\n",
    "#m.plot(x,y, 'bs', markersize=11, markeredgewidth=1, alpha=0.2)\n",
    "#m.scatter(x,y,c=MOD_DOY_array, s=150, marker='s',lw=0,cmap='Blues', alpha=.0)\n",
    "x,y=m(lon_start,lat_start)\n",
    "m.plot(x,y,'ro',markersize=4,markeredgewidth=0.0)\n",
    "#plt.title('Tb - std dev')\n",
    "#plt.colorbar()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "outFile = Path(outDir, 'test.png')\n",
    "\n",
    "# Plot Tb map\n",
    "#date='2019-05-08' # specify date of interest\n",
    "date = '2012-05-18'\n",
    "obs=0 # 0 for morning or 1 for evening observation\n",
    "\n",
    "# load Tb_std_dev array\n",
    "array=Tb_frame.loc[date].values\n",
    "array=array[obs]\n",
    "GRD=Tb_frame_GRD.loc[date].values\n",
    "GRD=GRD[obs]\n",
    "\n",
    "fig = plt.figure(figsize=(8,8))\n",
    "# create figure and axes instances\n",
    "#fig = plt.figure()\n",
    "ax = fig.add_axes([0.1,0.1,0.8,0.8])\n",
    "#ax = fig.add_subplot(121)\n",
    "# create polar stereographic Basemap instance.\n",
    "m = Basemap(projection='stere',lon_0=lon_start,lat_0=lat_start,lat_ts=90.,\\\n",
    "            llcrnrlat=(lat_start-.7),urcrnrlat=(lat_start+.7),\\\n",
    "            llcrnrlon=(lon_start-.7),urcrnrlon=(lon_start+.7),\\\n",
    "            rsphere=6371200.,resolution='l',area_thresh=10000, epsg=3857)\n",
    "m.drawcoastlines()\n",
    "m.drawcountries()\n",
    "m.drawstates()\n",
    "#m.etopo(scale=5, alpha=0.7)\n",
    "#m.bluemarble(scale=1)\n",
    "parallels = np.arange(0.,90,0.5)\n",
    "m.drawparallels(parallels,labels=[1,0,0,0],fontsize=10)\n",
    "# draw meridians\n",
    "meridians = np.arange(180.,360.,0.5)\n",
    "m.drawmeridians(meridians,labels=[0,0,0,1],fontsize=10)\n",
    "m.arcgisimage(service='World_Physical_Map', xpixels = 1500, verbose= True)\n",
    "\n",
    "x,y=m(pixel_lons, pixel_lats)\n",
    "#m.plot(x,y, 'bs', markersize=11, markeredgewidth=1, alpha=0.2)\n",
    "m.scatter(x,y,c=array, s=100, marker='s',lw=0, cmap='Greys', alpha=.8)  # cmap='Blues'\n",
    "#x,y=m(ISAlons,ISAlats)\n",
    "#m.plot(x,y,'ro',markersize=2,markeredgewidth=0.0)\n",
    "plt.title('Tb_'+date+' '+str(obs))\n",
    "\n",
    "# plot a red line on colorbar for the MOD for the GRD pixel\n",
    "cbar=plt.colorbar()\n",
    "cbar.ax.hlines(cbar.norm(GRD), 0, 1, color='red') \n",
    "plt.clim(230,250)  #color bar limits\n",
    "\n",
    "plt.show()\n",
    "plt.savefig(outFile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "outFile = Path(outDir, '5-8-19testTB2.png')\n",
    "\n",
    "ax = fig.add_axes([0.1,0.1,0.8,0.8])\n",
    "\n",
    "# create polar stereographic Basemap instance.\n",
    "m = Basemap(projection='stere',lon_0=lon_start,lat_0=lat_start,lat_ts=90.,\\\n",
    "            llcrnrlat=(lat_start-.7),urcrnrlat=(lat_start+.7),\\\n",
    "            llcrnrlon=(lon_start-.7),urcrnrlon=(lon_start+.7),\\\n",
    "            rsphere=6371200.,resolution='l',area_thresh=10000, epsg=3857)\n",
    "m.drawcoastlines()\n",
    "m.drawcountries()\n",
    "m.drawstates()\n",
    "#m.etopo(scale=5, alpha=0.7)\n",
    "#m.bluemarble(scale=1)\n",
    "parallels = np.arange(0.,90,0.5)\n",
    "m.drawparallels(parallels,labels=[1,0,0,0],fontsize=10)\n",
    "# draw meridians\n",
    "meridians = np.arange(180.,360.,0.5)\n",
    "m.drawmeridians(meridians,labels=[0,0,0,1],fontsize=10)\n",
    "m.arcgisimage(service='World_Physical_Map', xpixels = 1500, verbose= True)\n",
    "\n",
    "#x,y=m(pixel_lons, pixel_lats)\n",
    "#m.plot(x,y, 'bs', markersize=11, markeredgewidth=1, alpha=0.2)\n",
    "#m.scatter(x,y,c=MOD_DOY_array, s=150, marker='s',lw=0,cmap='Blues', alpha=.0)\n",
    "#x,y=m(ISAlons,ISAlats)\n",
    "#m.plot(x,y,'ro',markersize=2,markeredgewidth=0.0)\n",
    "#plt.title('Tb - std dev')\n",
    "#plt.colorbar()\n",
    "plt.show()\n",
    "plt.savefig(outFile)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "# PLOT THE EARLY EVENTS MAP\n",
    "# if getting errors, likely because there are 0 or only 1 early melt event for the subset\n",
    "\n",
    "fig = plt.figure(figsize=(8,8))\n",
    "# create figure and axes instances\n",
    "#fig = plt.figure()\n",
    "ax = fig.add_axes([0.1,0.1,0.8,0.8])\n",
    "#ax = fig.add_subplot(121)\n",
    "# create polar stereographic Basemap instance.\n",
    "m = Basemap(projection='stere',lon_0=lon_start,lat_0=lat_start,lat_ts=90.,\\\n",
    "            llcrnrlat=(lat_start-.7),urcrnrlat=(lat_start+.7),\\\n",
    "            llcrnrlon=(lon_start-.7),urcrnrlon=(lon_start+.7),\\\n",
    "            rsphere=6371200.,resolution='l',area_thresh=10000, epsg=3857)\n",
    "m.drawcoastlines()\n",
    "m.drawcountries()\n",
    "m.drawstates()\n",
    "#m.etopo(scale=5, alpha=0.7)\n",
    "#m.bluemarble(scale=1)\n",
    "parallels = np.arange(0.,90,5.)\n",
    "m.drawparallels(parallels,labels=[1,0,0,0],fontsize=10)\n",
    "# draw meridians\n",
    "meridians = np.arange(180.,360.,5.)\n",
    "m.drawmeridians(meridians,labels=[0,0,0,1],fontsize=10)\n",
    "m.arcgisimage(service='World_Physical_Map', xpixels = 1500, verbose= True)\n",
    "\n",
    "x,y=m(pixel_lons, pixel_lats)\n",
    "#m.plot(x,y, 'bs', markersize=11, markeredgewidth=1, alpha=0.2)\n",
    "m.scatter(x,y,c=events_array, s=150, marker='s',lw=0,cmap='RdBu', alpha=.8)\n",
    "#x,y=m(ISAlons,ISAlats)\n",
    "#m.plot(x,y,'ro',markersize=2,markeredgewidth=0.0)\n",
    "plt.title(sensor_SIR+'-'+Site+' - Early Melt Events - (Avg '+str(Years[0])+'-'+str(Years[-1])+')')\n",
    "plt.colorbar()\n",
    "plt.show()\n",
    "\n",
    "fig = plt.figure(figsize=(8,8))\n",
    "ax = fig.add_axes([0.1,0.1,0.8,0.8])\n",
    "\n",
    "# create polar stereographic Basemap instance.\n",
    "m = Basemap(projection='stere',lon_0=lon_start,lat_0=lat_start,lat_ts=90.,\\\n",
    "            llcrnrlat=(lat_start-.7),urcrnrlat=(lat_start+.7),\\\n",
    "            llcrnrlon=(lon_start-.7),urcrnrlon=(lon_start+.7),\\\n",
    "            rsphere=6371200.,resolution='l',area_thresh=10000, epsg=3857)\n",
    "m.drawcoastlines()\n",
    "m.drawcountries()\n",
    "m.drawstates()\n",
    "#m.etopo(scale=5, alpha=0.7)\n",
    "#m.bluemarble(scale=1)\n",
    "parallels = np.arange(0.,90,0.5)\n",
    "m.drawparallels(parallels,labels=[1,0,0,0],fontsize=10)\n",
    "# draw meridians\n",
    "meridians = np.arange(180.,360.,0.5)\n",
    "m.drawmeridians(meridians,labels=[0,0,0,1],fontsize=10)\n",
    "m.arcgisimage(service='World_Physical_Map', xpixels = 1500, verbose= True)\n",
    "\n",
    "#x,y=m(pixel_lons, pixel_lats)\n",
    "#m.plot(x,y, 'bs', markersize=11, markeredgewidth=1, alpha=0.2)\n",
    "#m.scatter(x,y,c=MOD_DOY_array, s=150, marker='s',lw=0,cmap='Blues', alpha=.0)\n",
    "#x,y=m(ISAlons,ISAlats)\n",
    "#m.plot(x,y,'ro',markersize=2,markeredgewidth=0.0)\n",
    "plt.title(sensor_SIR+'-'+Site+' - Early Melt Events - (Avg '+str(Years[0])+'-'+str(Years[-1])+')')\n",
    "#plt.colorbar()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rows_cols_env"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# returns the average Jan-Feb DAV for each pixel for each year - SIR\n",
    "Jan_Feb_DAV=Winter_DAV(data_SIR['TB'], data_SIR['cal_date'], data_SIR['cal_year'], subYears, rows_cols_env)\n",
    "Jan_Feb_DAV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Jan_Feb_DAV_avg=Jan_Feb_DAV.mean(axis=0)\n",
    "Jan_Feb_DAV_avg=Jan_Feb_DAV_avg.values\n",
    "#Jan_Feb_DAV_avg=np.round(Jan_Feb_DAV_avg)\n",
    "Jan_Feb_DAV_avg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# returns the average Jan-Feb DAV for the GRD pixel for each year\n",
    "Jan_Feb_DAV_GRD=Winter_DAV(data_GRD['TB'], data_GRD['cal_date'], data_GRD['cal_year'], subYears, rows_cols_GRD)\n",
    "Jan_Feb_DAV_GRD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# avg Jan-Feb DAV for all years - GRD pixel\n",
    "Jan_Feb_DAV_GRD_all=np.nanmean(Jan_Feb_DAV_GRD)\n",
    "Jan_Feb_DAV_GRD_all"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# PLOT WINTER DAV MAP\n",
    "# if getting errors, likely because there are 0 or only 1 early melt event for the subset\n",
    "fig = plt.figure(figsize=(8,8))\n",
    "# create figure and axes instances\n",
    "#fig = plt.figure()\n",
    "ax = fig.add_axes([0.1,0.1,0.8,0.8])\n",
    "#ax = fig.add_subplot(121)\n",
    "# create polar stereographic Basemap instance.\n",
    "m = Basemap(projection='stere',lon_0=lon_start,lat_0=lat_start,lat_ts=90.,\\\n",
    "            llcrnrlat=(lat_start-.7),urcrnrlat=(lat_start+.7),\\\n",
    "            llcrnrlon=(lon_start-.7),urcrnrlon=(lon_start+.7),\\\n",
    "            rsphere=6371200.,resolution='l',area_thresh=10000, epsg=3857)\n",
    "m.drawcoastlines()\n",
    "m.drawcountries()\n",
    "m.drawstates()\n",
    "#m.etopo(scale=5, alpha=0.7)\n",
    "#m.bluemarble(scale=1)\n",
    "parallels = np.arange(0.,90,0.5)\n",
    "m.drawparallels(parallels,labels=[1,0,0,0],fontsize=10)\n",
    "# draw meridians\n",
    "meridians = np.arange(180.,360.,0.5)\n",
    "m.drawmeridians(meridians,labels=[0,0,0,1],fontsize=10)\n",
    "m.arcgisimage(service='World_Physical_Map', xpixels = 1500, verbose= True)\n",
    "\n",
    "x,y=m(data_SIR['longitude'],data_SIR['latitude'])\n",
    "#m.plot(x,y, 'bs', markersize=11, markeredgewidth=1, alpha=0.2)\n",
    "m.scatter(x,y,c=Jan_Feb_DAV_avg, s=100, marker='s',lw=0,cmap='Blues', alpha=.8)\n",
    "#x,y=m(ISAlons,ISAlats)\n",
    "#m.plot(x,y,'ro',markersize=2,markeredgewidth=0.0)\n",
    "plt.title(sensor_SIR+'-'+Site+' - DAV - (Avg '+str(Years[0])+'-'+str(Years[-1])+')')\n",
    "\n",
    "# plot a red line on colorbar for the GRD mean - all years\n",
    "cbar=plt.colorbar()\n",
    "cbar.ax.hlines(cbar.norm(Jan_Feb_DAV_GRD_all), 0, 1, color='green') \n",
    "\n",
    "plt.show()\n",
    "\n",
    "fig = plt.figure(figsize=(8,8))\n",
    "ax = fig.add_axes([0.1,0.1,0.8,0.8])\n",
    "\n",
    "# create polar stereographic Basemap instance.\n",
    "m = Basemap(projection='stere',lon_0=lon_start,lat_0=lat_start,lat_ts=90.,\\\n",
    "            llcrnrlat=(lat_start-.7),urcrnrlat=(lat_start+.7),\\\n",
    "            llcrnrlon=(lon_start-.7),urcrnrlon=(lon_start+.7),\\\n",
    "            rsphere=6371200.,resolution='l',area_thresh=10000, epsg=3857)\n",
    "m.drawcoastlines()\n",
    "m.drawcountries()\n",
    "m.drawstates()\n",
    "#m.etopo(scale=5, alpha=0.7)\n",
    "#m.bluemarble(scale=1)\n",
    "parallels = np.arange(0.,90,0.5)\n",
    "m.drawparallels(parallels,labels=[1,0,0,0],fontsize=10)\n",
    "# draw meridians\n",
    "meridians = np.arange(180.,360.,0.5)\n",
    "m.drawmeridians(meridians,labels=[0,0,0,1],fontsize=10)\n",
    "m.arcgisimage(service='World_Physical_Map', xpixels = 1500, verbose= True)\n",
    "\n",
    "#x,y=m(pixel_lons, pixel_lats)\n",
    "#m.plot(x,y, 'bs', markersize=11, markeredgewidth=1, alpha=0.2)\n",
    "#m.scatter(x,y,c=MOD_DOY_array, s=150, marker='s',lw=0,cmap='Blues', alpha=.0)\n",
    "#x,y=m(ISAlons,ISAlats)\n",
    "#m.plot(x,y,'ro',markersize=2,markeredgewidth=0.0)\n",
    "plt.title(sensor_SIR+'-'+Site+' - DAV - (Avg '+str(Years[0])+'-'+str(Years[-1])+')')\n",
    "#plt.colorbar()\n",
    "plt.show()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# winter DAV histogram\n",
    "plt.close()\n",
    "fig,ax=plt.subplots()\n",
    "ax.hist(Jan_Feb_DAV_avg)\n",
    "avg=np.mean(Jan_Feb_DAV_avg)  # calculate average winter DAV all pixels\n",
    "ax.axvline(x=avg, color='blue', linestyle='dashed')  # plot a line for the average winter DAV for all pixels\n",
    "ax.axvline(x=Jan_Feb_DAV_GRD_all, color='red')  # plot a line for winter DAV for GRD pixel\n",
    "ax.set_xlabel('DAV (K)')\n",
    "ax.set_ylabel('Count')\n",
    "ax.set_title('Winter (Jan-Feb) DAV')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "SevZ_AWS#need to run this cell for the max DAV to work\n",
    "y_dims_list=list(range(len(data_SIR['TB'][0,:,0])))  # creates a list of the y-dimension pixel indices, used for plotting\n",
    "x_dims_list=list(range(len(data_SIR['TB'][0,0,:])))\t# creates a list of the x-dimension pixel indices, for plotting\n",
    "y_s=list(range(rows_cols_env[0],rows_cols_env[1]))  # makes a list of the y(row) numbers so the dataframe of early melt events can be indexed\n",
    "x_s=list(range(rows_cols_env[2],rows_cols_env[3]))  # makes a list of the x(col) numbers so the dataframe of early melt events can be indexed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create array of the max DAV for each pixel for all years\n",
    "DAV_frame=pd.DataFrame()\n",
    "for i in y_dims_list:\n",
    "    for j in x_dims_list:\n",
    "        column=pd.DataFrame(data=DAV_SIR[:,i,j], columns=[str(y_s[i])+','+str(x_s[j])])\n",
    "        DAV_frame=pd.concat([DAV_frame,column],axis=1)\n",
    "\n",
    "DAV_frame.rename_axis(columns=\"Row,Col\", inplace=True)\n",
    "DAV_frame['date'] = data_SIR['cal_date']\n",
    "DAV_frame.set_index('date', inplace=True)\n",
    "#DAV_frame=DAV_frame.groupby(pd.Grouper(freq='A')).max() #note to joan might also do sum for the melt flags\n",
    "DAV_max_array=DAV_frame.max().values\n",
    "DAV_max_array"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "DAV_frame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create array of the max DAV for each pixel for year of interest\n",
    "year=2000  #specify year\n",
    "DAV_max_array_year=DAV_frame.loc[str(year)].max().values\n",
    "DAV_max_array_year"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "DAV_max_GRD=np.nanmax(DAV_GRD)\n",
    "DAV_max_GRD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "DAV_GRD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "year"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Note this year needs to be the same as the max dav plot below\n",
    "#year=2005\n",
    "DAV_max_GRD_year=np.nanmax(DAV_GRD[data_GRD['cal_year']==year])\n",
    "DAV_max_GRD_year"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# PLOT MAX DAV MAP\n",
    "#plot_type='all'\n",
    "year = 2000\n",
    "plot_type=year\n",
    "#NOTE YEAR SPECIFIED UP A FEW LEVELS (~3)\n",
    "#year=2005\n",
    "\n",
    "if plot_type=='all':\n",
    "    array=DAV_max_array\n",
    "    title=sensor_SIR+'-'+Site+' - Max DAV (K) - (Avg of '+str(Years[0])+'-'+str(Years[-1])+')'\n",
    "    GRD=DAV_max_GRD\n",
    "elif plot_type==year:\n",
    "\n",
    "    array=DAV_max_array_year\n",
    "    GRD=DAV_max_GRD_year\n",
    "    title=sensor_SIR+'-'+Site+' - Max DAV (K) - ('+str(year)+')'\n",
    "\n",
    "fig = plt.figure(figsize=(8,8))\n",
    "# create figure and axes instances\n",
    "#fig = plt.figure()\n",
    "ax = fig.add_axes([0.1,0.1,0.8,0.8])\n",
    "#ax = fig.add_subplot(121)\n",
    "# create polar stereographic Basemap instance.\n",
    "m = Basemap(projection='stere',lon_0=lon_start,lat_0=lat_start,lat_ts=90.,\\\n",
    "            llcrnrlat=(lat_start-.7),urcrnrlat=(lat_start+.7),\\\n",
    "            llcrnrlon=(lon_start-.7),urcrnrlon=(lon_start+.7),\\\n",
    "            rsphere=6371200.,resolution='l',area_thresh=10000, epsg=3857)\n",
    "m.drawcoastlines()\n",
    "m.drawcountries()\n",
    "m.drawstates()\n",
    "#m.etopo(scale=5, alpha=0.7)\n",
    "#m.bluemarble(scale=1)\n",
    "parallels = np.arange(0.,90,0.5)\n",
    "m.drawparallels(parallels,labels=[1,0,0,0],fontsize=10)\n",
    "# draw meridians\n",
    "meridians = np.arange(180.,360.,0.5)\n",
    "m.drawmeridians(meridians,labels=[0,0,0,1],fontsize=10)\n",
    "m.arcgisimage(service='World_Physical_Map', xpixels = 1500, verbose= True)\n",
    "\n",
    "x,y=m(data_SIR['longitude'], data_SIR['latitude'])\n",
    "      # formerly pixel_lons, pixel_lats)\n",
    "#m.plot(x,y, 'bs', markersize=11, markeredgewidth=1, alpha=0.2)\n",
    "m.scatter(x,y,c=array, s=100, marker='s',lw=0,cmap='Blues', alpha=.8)\n",
    "#x,y=m(ISAlons,ISAlats)\n",
    "#m.plot(x,y,'ro',markersize=2,markeredgewidth=0.0)\n",
    "plt.title(title)\n",
    "\n",
    "# plot a red line on colorbar for the GRD max\n",
    "cbar=plt.colorbar()\n",
    "#cbar.ax.hlines(cbar.norm(DAV_max_GRD), 0, 1, color='red') \n",
    "cbar.ax.hlines(cbar.norm(DAV_max_GRD_year), 0, 1, color='red')\n",
    "plt.show()\n",
    "\n",
    "fig = plt.figure(figsize=(8,8))\n",
    "ax = fig.add_axes([0.1,0.1,0.8,0.8])\n",
    "\n",
    "# create polar stereographic Basemap instance.\n",
    "m = Basemap(projection='stere',lon_0=lon_start,lat_0=lat_start,lat_ts=90.,\\\n",
    "            llcrnrlat=(lat_start-.7),urcrnrlat=(lat_start+.7),\\\n",
    "            llcrnrlon=(lon_start-.7),urcrnrlon=(lon_start+.7),\\\n",
    "            rsphere=6371200.,resolution='l',area_thresh=10000, epsg=3857)\n",
    "m.drawcoastlines()\n",
    "m.drawcountries()\n",
    "m.drawstates()\n",
    "#m.etopo(scale=5, alpha=0.7)\n",
    "#m.bluemarble(scale=1)\n",
    "parallels = np.arange(0.,90,0.5)\n",
    "m.drawparallels(parallels,labels=[1,0,0,0],fontsize=10)\n",
    "# draw meridians\n",
    "meridians = np.arange(180.,360.,0.5)\n",
    "m.drawmeridians(meridians,labels=[0,0,0,1],fontsize=10)\n",
    "m.arcgisimage(service='World_Physical_Map', xpixels = 1500, verbose= True)\n",
    "\n",
    "#x,y=m(pixel_lons, pixel_lats)\n",
    "#m.plot(x,y, 'bs', markersize=11, markeredgewidth=1, alpha=0.2)\n",
    "#m.scatter(x,y,c=MOD_DOY_array, s=150, marker='s',lw=0,cmap='Blues', alpha=.0)\n",
    "#x,y=m(ISAlons,ISAlats)\n",
    "#m.plot(x,y,'ro',markersize=2,markeredgewidth=0.0)\n",
    "plt.title(title)\n",
    "#plt.colorbar()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Histograms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "outFile = Path(outDir, 'SFF_Hist2002-2004.png')\n",
    "\n",
    "# generate histogram - SIR - pick year \n",
    "year=2000  # specify year here\n",
    "#Tb_threshold=252  #specify threshold be careful it was set above by sensor\n",
    "data = data_SIR['TB'][data_SIR['cal_year']==year] # CETB_data for all pixels in subset\n",
    "data = data[data>=0]\n",
    "bins = range(150,300)  # bins for histogram\n",
    "fig,ax=plt.subplots()\n",
    "ax.axvline(x=Tb_threshold, color='red')\n",
    "ax.hist(data, bins)\n",
    "ax.set_title(str(year)+ ' '+Site+' SIR Histogram')\n",
    "#ax.set_title(prefix+'row'+str(x)+'col'+str(y))\n",
    "ax.set_xlabel('Brightness Temp (K)')\n",
    "#fig.savefig('/home/mij216/Documents/GLaIL/'+prefix+'row'+str(x)+'col'+str(y)+'.png')\n",
    "plt.savefig(outFile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "outFile = Path(outDir, 'FF_HistSIR2002-2004.png')\n",
    "\n",
    "# generate histogram - SIR - all data\n",
    "#year=2005\n",
    "#Tb_threshold=252  #set above by sensor\n",
    "data = data_SIR['TB'][:] # CETB_data for all pixels in subset\n",
    "data = data[data>=0]\n",
    "bins = range(150,300)  # bins for histogram\n",
    "fig,ax=plt.subplots()\n",
    "ax.hist(data, bins)\n",
    "ax.set_title('All data SIR Histogram'+' '+Site)\n",
    "ax.axvline(x=Tb_threshold, color='red')\n",
    "#ax.set_title(prefix+'row'+str(x)+'col'+str(y))\n",
    "ax.set_xlabel('Brightness Temp (K)')\n",
    "#fig.savefig('/home/mij216/Documents/GLaIL/'+prefix+'row'+str(x)+'col'+str(y)+'.png')\n",
    "plt.savefig(outFile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "outFile = Path(outDir, 'FF_HistGRD2002-2004.png')\n",
    "               \n",
    "# generate histogram - GRD - all data\n",
    "data = data_GRD['TB'][:] # CETB_data for all pixels in subset\n",
    "data = data[data>=0]\n",
    "bins = range(150,300)  # bins for histogram\n",
    "fig,ax=plt.subplots()\n",
    "ax.hist(data, bins)\n",
    "ax.set_title('All data GRD Histogram')\n",
    "ax.axvline(x=Tb_threshold, color='red')\n",
    "#ax.set_title(prefix+'row'+str(x)+'col'+str(y))\n",
    "ax.set_xlabel('Brightness Temp (K)')\n",
    "#fig.savefig('/home/mij216/Documents/GLaIL/'+prefix+'row'+str(x)+'col'+str(y)+'.png')\n",
    "plt.savefig(outFile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# min/max/avg histogram of the 64 SIR pixels, plus the GRD pixel\n",
    "year=2004\n",
    "title=str(lat_start)+'_'+str(lon_start)\n",
    "frame=min_max_series(data_GRD['TB'], data_SIR['TB'], data_SIR['cal_date'], data_SIR['cal_year'], year, title)\n",
    "frame['date'] = data_SIR['cal_date']\n",
    "frame.set_index('date', inplace=True)\n",
    "frame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#some kind of numpy pandas compatibility issue\n",
    "frame.index\n",
    "frame['min'].plot.hist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "outFile = Path(outDir, 'test_SIRhistdis02-04.png')\n",
    "frame.plot.hist(alpha=0.8, bins=100, histtype='step', ylim=[0,100], xlim=[150,320], title=title)\n",
    "plt.savefig(outFile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "year=2003"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "outFile = Path(outDir, 'FF_timeseriesplot2003.png')\n",
    "\n",
    "# min/max/avg time series  of the 64 SIR pixels, plus the GRD pixel\n",
    "# plot time series of min/max/mean of the 3 km pixels in subset and the 25km (GRD) pixel that envelopes them    \n",
    "frame.plot(xlim=[str(year)+'-01-01',str(year)+'-12-31'], ylim=[150,320], title=title)\n",
    "plt.savefig(outFile)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## TIME SERIES"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Tb and DAV time series for GRD pixel, pick one year\n",
    "year=2003\n",
    "#Tb_threshold=252\n",
    "#DAV_threshold=18\n",
    "TbDAV_series_one_year(data_GRD['TB'],DAV_GRD, data_GRD['cal_date'], data_GRD['cal_year'], year, Tb_threshold, DAV_threshold)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Tb and DAV time series for SIR pixels, pick one year\n",
    "#year=2003\n",
    "#Tb_threshold=252\n",
    "#DAV_threshold=18\n",
    "outFile = Path(outDir, 'FFtimeseriesALLSIRplots2003.png')\n",
    "TbDAV_series_one_year(data_SIR['TB'], DAV_SIR, data_SIR['cal_date'], data_SIR['cal_year'], year, Tb_threshold, DAV_threshold)\n",
    "plt.savefig(outFile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Tb and DAV time series of the single SIR pixel closest to the user-specified lat/lon at top of notebook\n",
    "#year=2019\n",
    "year=2003\n",
    "#Tb_threshold=252\n",
    "#DAV_threshold=18\n",
    "data=read_Tb_whole(datadir_SIR, prefix_SIR, subYears,\n",
    "                   y_start=rows_cols[0][0],\n",
    "                   y_end=rows_cols[0][0]+1,\n",
    "                   x_start=rows_cols[1][0],\n",
    "                   x_end=rows_cols[1][0]+1)\n",
    "\n",
    "SIR_1px=data['TB']\n",
    "DAV_1px=calc_DAV(SIR_1px)\n",
    "TbDAV_series_one_year(SIR_1px, DAV_1px, data['cal_date'], data['cal_year'], year, Tb_threshold, DAV_threshold) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## CREATE CSVs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "col_1px = [\"%d,%d\" % (rows_cols[0][0], rows_cols[1][0])]\n",
    "TB_SIR_1px=pd.DataFrame(SIR_1px.flatten(), \n",
    "                        columns=col_1px, \n",
    "                        index=data['cal_date'])\n",
    "TB_SIR_1px"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "DAV_SIR_1px=pd.DataFrame(DAV_1px.flatten(),\n",
    "                         columns=col_1px,\n",
    "                         index=data['cal_date'])\n",
    "DAV_SIR_1px"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['cal_date'], data_GRD['cal_date']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# CREATE CSV of GRD and SIR TB, GRD and SIR64 ST, and GRD And SIR DAV\n",
    "\n",
    "# CREATE CSV OF TB, DAV, SD and sirSD FOR THE GRD PIXEL - ALL YEARS\n",
    "Tb_frame = pd.DataFrame(data_GRD['TB'][:,0,0], index=data_GRD['cal_date'])\n",
    "DAV_frame = pd.DataFrame(DAV_GRD[:,0,0], index=data_GRD['cal_date'])\n",
    "GRD_data_out = pd.concat([Tb_frame,DAV_frame,TB_SIR_1px,DAV_SIR_1px,Tb_sd_frame_GRD,Tb_std_64,Tb_sd_diff],\n",
    "                         axis=1)\n",
    "GRD_data_out.columns = ['GRDTb36V','GRDDAV36V','SIRTB36V','SIRDAV36V',\n",
    "                        'GRD_SD_36V','SIR64_to_SD_36V','SD_diff[GRD-SIR]']\n",
    "GRD_data_out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "outFile = Path(outNoisy, 'sd', \n",
    "               sat_GRD+'_'+sensor_GRD+'-'+channel_GRD+'-'+alg_GRD+'_'\n",
    "               +str(lat_start)+'_'+str(lon_start)+'SDDATA.csv')  # modify destination to save csv\n",
    "GRD_data_out.to_csv(outFile)\n",
    "outFile"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# CREATE CSV OF TB FOR THE GRD PIXEL - ALL YEARS\n",
    "Tb_frame=pd.DataFrame(data_GRD['TB'][:,0,0], index=data_GRD['cal_date'])\n",
    "outFile = Path(outNoisy, 'noisytb_csv',\n",
    "               sat_GRD+'_'+sensor_GRD+'-'+channel_GRD+'-'+alg_GRD+'_'\n",
    "               +str(lat_start)+'_'+str(lon_start)+'GRDTB.csv')  # modify destination to save csv\n",
    "Tb_frame.to_csv(outFile)\n",
    "outFile"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# CREATE CSV of DAV FOR THE GRD PIXEL - ALL YEARS\n",
    "DAV_frame=pd.DataFrame(DAV_GRD[:,0,0], index=data['cal_date'])\n",
    "outFile = Path(outNoisy, 'noisydav_csv',\n",
    "               sat_GRD+'_'+sensor_GRD+'-'+channel_GRD+'-'+alg_GRD+'_'\n",
    "               +str(lat_start)+'_'+str(lon_start)+'GRDDAV.csv') # modify destination to save csv\n",
    "DAV_frame.to_csv(outFile)\n",
    "outFile"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# CREATE CSV of TB FOR 64pixel subset OF SIR PIXELS - ALL YEARS\n",
    "Tb_frame=pd.DataFrame()\n",
    "for i in y_dims_list:\n",
    "    for j in x_dims_list:\n",
    "        column=pd.DataFrame(data=data_SIR['TB'][:,i,j], columns=[str(y_s[i])+','+str(x_s[j])])\n",
    "        Tb_frame=pd.concat([Tb_frame,column],axis=1)\n",
    "Tb_frame=Tb_frame.set_index(data_SIR['cal_date'])\n",
    "outFile = Path(outNoisy, 'noisytbSIR_csv',\n",
    "               sat_GRD+'_'+sensor_GRD+'-'+channel_GRD+'-'+alg_GRD+'_'\n",
    "               +str(lat_start)+'_'+str(lon_start)+'SIRTB.csv')  # modify destination to save csv\n",
    "Tb_frame.to_csv(outFile)\n",
    "outFile"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# CREATE CSV of DAV FOR 64pixel subset OF SIR PIXELS - ALL YEARS\n",
    "DAV_frame=pd.DataFrame()\n",
    "for i in y_dims_list:\n",
    "    for j in x_dims_list:\n",
    "        column=pd.DataFrame(data=DAV_SIR[:,i,j], columns=[str(y_s[i])+','+str(x_s[j])])\n",
    "        DAV_frame=pd.concat([DAV_frame,column],axis=1)\n",
    "DAV_frame=DAV_frame.set_index(data_SIR['cal_date'])\n",
    "outFile = Path(outNoisy, 'noisydavSIR_csv',\n",
    "               sat_GRD+'_'+sensor_GRD+'-'+channel_GRD+'-'+alg_GRD+'_'\n",
    "               +str(lat_start)+'_'+str(lon_start)+'SIRDAV.csv')  # modify destination to save csv\n",
    "DAV_frame.to_csv(outFile)\n",
    "outFile\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}