Develop diagnostics for FV3 LAMDA

LAMDA/LAMDA_tools.py

@@ -0,0 +1,56 @@
+import numpy as np
+import pandas as pd
+
+
+def land_fraction(self, df):
+    """
+    Returns dataframe for only land data.
+    """
+    df = df[df['land_fraction'] > 0]
+
+    return df
+
+
+def water_fraction(self, df):
+    """
+    Returns dataframe for only water data.
+    """
+    df = df[df['water_fraction'] > 0]
+
+    return df
+
+
+def cloud_fraction(self, df):
+    """
+    Returns dataframe for only cloud data.
+    """
+    df = df[df['cloud_fraction'] > 0]
+
+    return df
+
+
+def vegetation_fraction(self, df):
+    """
+    Returns dataframe for only vegetation data.
+    """
+    df = df[df['vegetation_fraction'] > 0]
+
+    return df
+
+
+def ice_fraction(self, df):
+    """
+    Returns dataframe for only ice data.
+    """
+    df = df[df['ice_fraction'] > 0]
+
+    return df
+
+
+def snow_fraction(self, df):
+    """
+    Returns dataframe for only snow data.
+    """
+    df = df[df['snow_fraction'] > 0]
+
+    return df

LAMDA/bias_rmse_timeseries.py

@@ -0,0 +1,202 @@
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+from pyGSI.gsi_stat import GSIstat
+from emcpy.plots.plots import LinePlot, HorizontalLine
+from emcpy.plots import CreatePlot
+
+
+def _get_conventional_data(df, config):
+    """
+    Grabs appropriate data from the dataframe
+    and creates EMCPy plot list, title and
+    save file name for conventional data.
+    """
+
+    indx = df.index.get_level_values('stat') == ''
+    bias_indx = np.ma.logical_or(indx, df.index.get_level_values('stat') == 'bias')
+    rmse_indx = np.ma.logical_or(indx, df.index.get_level_values('stat') == 'rms')
+    bias_df = df.iloc[bias_indx].reset_index()
+    rmse_df = df.iloc[rmse_indx].reset_index()
+
+    data_col = rmse_df.columns[-1]
+
+    # Grab data from dataframe
+    cycles = rmse_df['date'].dt.strftime("%d %b\n%Y %Hz")
+    rmse = rmse_df[data_col]
+    bias = bias_df[data_col]
+
+    # Create bias object
+    bias_line = LinePlot(cycles, bias)
+    bias_line.label = 'Bias'
+    bias_line.marker = 'o'
+    bias_line.markersize = 3
+
+    # Create rmse
+    rmse_line = LinePlot(cycles, rmse)
+    rmse_line.color = 'tab:green'
+    rmse_line.label = 'RMSE'
+    rmse_line.marker = 'o'
+    rmse_line.markersize = 3
+
+    # Create a line at 0
+    zero_line = HorizontalLine(y=0)
+
+    # Add objects to plot list
+    plot_list = [bias_line, rmse_line, zero_line]
+
+    # Create title
+    title = (f"{config['experiment name']} {config['bias type']} "
+             f"RMSE and Bias Time Series\nOb Type: {config['ob type']} "
+             f"Type: {config['obsid']} Subtype: {config['subtype']} "
+             f"tm0{config['tm']}")
+    config['title'] = title
+
+    # Create save file name
+    save_cycles = bias_df['date'].dt.strftime('%Y%m%d%H').to_numpy()
+    savefile = (f"{save_cycles[0]}_{save_cycles[-1]}_{config['experiment name']}_"
+                f"{config['ob type']}_{config['obsid']}_{config['subtype']}_"
+                f"{config['bias type']}_tm0{config['tm']}_rmse_bias_timeseries.png")
+    config['save file'] = savefile
+
+    return plot_list, config
+
+
+def _get_radiance_data(df, config):
+    """
+    Grabs appropriate data from the dataframe
+    and creates EMCPy plot list, title and
+    save file name for radiance data.
+    """
+
+    df = df.reset_index()
+
+    # Grab data from dataframe
+    cycles = df['date'].dt.strftime("%d %b\n%Y %Hz")
+    omf_bc = df['OmF_bc']
+    omf_wobc = df['OmF_wobc']
+    rmse = df['rms']
+
+    # Create bias correction object
+    bc_line = LinePlot(cycles, omf_bc)
+    bc_line.label = 'Bias w/ BC'
+    bc_line.marker = 'o'
+    bc_line.markersize = 3
+    bc_line.linestyle = '-.'
+
+    # Create object without bias correction
+    wobc_line = LinePlot(cycles, omf_wobc)
+    wobc_line.color = 'tab:green'
+    wobc_line.label = 'Bias w/o BC'
+    wobc_line.marker = 'o'
+    wobc_line.markersize = 3
+    wobc_line.linestyle = '--'
+
+    # Create rmse
+    rmse_line = LinePlot(cycles, rmse)
+    rmse_line.color = 'tab:brown'
+    rmse_line.label = 'RMSE'
+    rmse_line.marker = 'o'
+    rmse_line.markersize = 3
+
+    # Create a line at 0
+    zero_line = HorizontalLine(y=0)
+
+    # Add objects to plot list
+    plot_list = [bc_line, wobc_line, rmse_line, zero_line]
+
+    # Create title
+    title = (f"{config['experiment name']} {config['bias type']} "
+             f"RMSE and Bias Time Series \n{config['sensor']} "
+             f"{config['satellite']} Channel {config['channel']} "
+             f"tm0{config['tm']}")
+    config['title'] = title
+
+    # Create save file name
+    save_cycles = df['date'].dt.strftime('%Y%m%d%H').to_numpy()
+
+    savefile = (f"{save_cycles[0]}_{save_cycles[-1]}_{config['experiment name']}_"
+                f"{config['sensor']}_{config['satellite']}_channel_{config['channel']}"
+                f"_{config['bias type']}_tm0{config['tm']}_rmse_bias_timeseries.png")
+    config['save file'] = savefile
+
+    return plot_list, config
+
+
+def _plot_bias_rmse_timeseries(df, config, outdir):
+    """
+    Used indexed df to plot rmse and bias.
+    """
+
+    if config['data type'] == 'radiance':
+        plot_list, config = _get_radiance_data(df, config)
+
+    elif config['data type'] == 'conventional':
+        plot_list, config = _get_conventional_data(df, config)
+
+    # Create plot and draw data
+    myplt = CreatePlot(figsize=(10, 6))
+    myplt.draw_data(plot_list)
+
+    # Add features
+    myplt.set_ylim(-5, 5)
+    myplt.add_grid(linewidth=0.5, color='grey', linestyle='--')
+    myplt.add_legend(loc='lower right', fontsize='large')
+
+    myplt.add_title(config['title'], fontsize=14)
+
+    # Return matplotlib figure and save
+    fig = myplt.return_figure()
+    fig.savefig(outdir + config['save file'], bbox_inches='tight',
+                pad_inches=0.1)
+    plt.close('all')
+
+
+def bias_rmse_timeseries(df, config, outdir):
+    """
+    Plots a timeseries of bias and rmse.
+
+    Args:
+        df : (pandas dataframe) multidimensional pandas dataframe
+             with several cycles of gsi stats data
+        config : (dict) dictionary including informaton about the data
+                 being plotted
+        outdir : (str) path to output figures
+    """
+
+    if config['data type'] == 'radiance':
+
+        # Select data by satellite and channel
+        for idx_col in ['satellite', 'channel']:
+            indx = df.index.get_level_values(idx_col) == ''
+            indx = np.ma.logical_or(indx, df.index.get_level_values(idx_col) == config[idx_col])
+            df = df.iloc[indx]
+
+    elif config['data type'] == 'conventional':
+        for idx_col in ['typ', 'use']:
+            d = {
+                'typ': config['obsid'],
+                'styp': config['subtype'],
+                'use': 'asm'
+            }
+
+            config_val = d[idx_col]
+            indx = df.index.get_level_values(idx_col) == ''
+            indx = np.ma.logical_or(indx, df.index.get_level_values(idx_col) == config_val)
+            df = df.iloc[indx]
+
+    # Create omf and oma df
+    indx = df.index.get_level_values('it') == ''
+    omf_indx = np.ma.logical_or(indx, df.index.get_level_values('it') == 1)
+    omf_df = df.iloc[omf_indx]
+
+    oma_indx = np.ma.logical_or(indx, df.index.get_level_values('it') == 3)
+    oma_df = df.iloc[oma_indx]
+
+    # Plot omf
+    config['bias type'] = 'OmF'
+    _plot_bias_rmse_timeseries(omf_df, config, outdir)
+
+    # Plot oma
+    config['bias type'] = 'OmA'
+    _plot_bias_rmse_timeseries(oma_df, config, outdir)

LAMDA/bias_stddev_channel.py

@@ -0,0 +1,126 @@
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+from pyGSI.gsi_stat import GSIstat
+from emcpy.plots.plots import LinePlot, HorizontalLine
+from emcpy.plots import CreatePlot
+
+
+def _plot_bias_stddev_channel(df, config, outdir):
+    """
+    Plot the bias and standard deviation per channel
+    for a single cycle and an average for multiple
+    cycles.
+    """
+    # Create single cycle and average dataframe
+    cycles = df.index.get_level_values(0).unique()
+    n_days = len(cycles)/2
+    scyc_df = df.loc[cycles[-1]]
+    avg_df = df.groupby(level=[1, 2, 3, 4]).mean()
+
+    # Get data
+    channels = df.index.get_level_values(-1).unique().to_numpy()
+    omf_bc = scyc_df['OmF_bc']
+    omf_std = scyc_df['std']
+    avg_bc = avg_df['OmF_bc']
+    avg_std = avg_df['std']
+
+    # Create bias correction object
+    bc_line = LinePlot(channels, omf_bc)
+    bc_line.color = 'tab:green'
+    bc_line.label = 'OmF'
+    bc_line.marker = 'o'
+    bc_line.markersize = 4
+    bc_line.linewidth = 3
+
+    # Create standard deviation object
+    std_dev = LinePlot(channels, omf_std)
+    std_dev.color = 'tab:orange'
+    std_dev.label = 'Std Dev'
+    std_dev.marker = 'o'
+    std_dev.markersize = 4
+    std_dev.linewidth = 3
+
+    # Create average bias correction object
+    avg_bc_line = LinePlot(channels, avg_bc)
+    avg_bc_line.color = 'tab:green'
+    avg_bc_line.label = f'{n_days} Day OmF Average'
+    avg_bc_line.marker = 'o'
+    avg_bc_line.markersize = 4
+    avg_bc_line.linewidth = 3
+    avg_bc_line.linestyle = '--'
+
+    # Create average standard deviation object
+    avg_std_dev = LinePlot(channels, avg_std)
+    avg_std_dev.color = 'tab:orange'
+    avg_std_dev.label = f'{n_days} Day Std Average'
+    avg_std_dev.marker = 'o'
+    avg_std_dev.markersize = 4
+    avg_std_dev.linewidth = 3
+    avg_std_dev.linestyle = '--'
+
+    # Create a line at 0
+    zero_line = HorizontalLine(y=0)
+    zero_line.linewidth = 1.25
+
+    # Create plot and draw data
+    myplt = CreatePlot(figsize=(10, 6))
+    plt_list = [bc_line, avg_bc_line, std_dev, avg_std_dev, zero_line]
+    myplt.draw_data(plt_list)
+
+    # Add features
+    myplt.set_ylim(-3, 3)
+    myplt.set_xticks(channels)
+    myplt.add_grid(linewidth=0.5, color='grey', linestyle='--')
+    myplt.add_legend(loc='lower right', fontsize='large')
+    myplt.add_xlabel('Channels')
+
+    str_cycles = cycles.strftime('%Y%m%d%H').to_numpy()
+    left_title = (f"{config['bias type']} (with Bias Correction) - Observed (K)"
+                  f"\n{config['sensor']}_{config['satellite']}")
+    right_title = f"{str_cycles[-1]}"
+    myplt.add_title(left_title, loc='left', fontsize=14)
+    myplt.add_title(right_title, loc='right', fontsize=12, fontweight='semibold')
+
+    # Return matplotlib figure
+    fig = myplt.return_figure()
+    savefile = (f"{str_cycles[-1]}_{config['sensor']}_{config['satellite']}"
+                f"_channel_{config['channel']}_{config['bias type']}"
+                f"_bias_std_channel.png")
+    fig.savefig('./' + savefile, bbox_inches='tight',
+                pad_inches=0.1)
+    plt.close('all')
+
+
+def bias_stddev_channel(df, config, outdir):
+    """
+    Plots bias and standard deviation per channel.
+
+    Args:
+        df : (pandas dataframe) multidimensional pandas dataframe
+             with several cycles of gsi stats data
+        config : (dict) dictionary including informaton about the data
+                 being plotted
+        outdir : (str) path to output figures
+    """
+    # Select data by satellite and channel
+    for idx_col in ['satellite']:
+        indx = df.index.get_level_values(idx_col) == ''
+        indx = np.ma.logical_or(indx, df.index.get_level_values(idx_col) == config[idx_col])
+        df = df.iloc[indx]
+
+    # Create omf and oma df
+    indx = df.index.get_level_values(idx_col) == ''
+    omf_indx = np.ma.logical_or(indx, df.index.get_level_values('it') == 1)
+    omf_df = df.iloc[omf_indx]
+
+    oma_indx = np.ma.logical_or(indx, df.index.get_level_values('it') == 3)
+    oma_df = df.iloc[oma_indx]
+
+    # Plot omf
+    config['bias type'] = 'Ges'
+    _plot_bias_stddev_channel(omf_df, config, outdir)
+
+    # Plot oma
+    config['bias type'] = 'Anl'
+    _plot_bias_stddev_channel(oma_df, config, outdir)