From 9d9909b8b06a435fffa4874ba6835ced7128cfbe Mon Sep 17 00:00:00 2001 From: lauratomkins Date: Tue, 5 Dec 2023 14:47:44 -0500 Subject: [PATCH 01/18] Create cfad.py --- pyart/retrieve/cfad.py | 66 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 pyart/retrieve/cfad.py diff --git a/pyart/retrieve/cfad.py b/pyart/retrieve/cfad.py new file mode 100644 index 0000000000..4f13ed063d --- /dev/null +++ b/pyart/retrieve/cfad.py @@ -0,0 +1,66 @@ +""" +Create CFAD from a radar or grid field + +""" + +import numpy as np + + +def create_cfad( + field_data, + altitude_data, + field_bins, + altitude_bins, + min_frac_thres=0.1, +): + """ + This function returns a Contoured Frequency by Altitude Diagram (CFAD; Yuter et al. 1995), a 2-dimensional + histogram that is normalized by the number of points at each altitude. Altitude bins are masked where the counts + are less than a minimum fraction of the largest number of counts for any altitude row. + + field_data : array + Array of radar data to use for CFAD calculation. + altitude_data : array + Array of corresponding altitude data to use for CFAD calculation. + Note: must be the same shape as `field_data` + field_bins : list + List of bin edges for field values to use for CFAD creation. + altitude_bins : list + List of bin edges for height values to use for CFAD creation. + min_frac_thres : float, optional + Fraction of values to remove in CFAD normalization (default 0.1). If an altitude row has a total count that + is less than min_frac_thres of the largest number of total counts for any altitude row, the bins in that + altitude row are masked. + Returns + ------- + freq_norm : array + Array of normalized frequency. + height_edges : array + Array of bin edges for height data. + field_edges : array of x coordinates + Array of bin edges for field data. + References + ---------- + Yuter, S. E., and R. A. Houze, 1995: Three-Dimensional Kinematic and + Microphysical Evolution of Florida Cumulonimbus. Part II: Frequency Distributions + of Vertical Velocity, Reflectivity, and Differential Reflectivity. Mon. Wea. Rev. + 123, 1941-1963. https://doi.org/10.1175/1520-0493(1995)123%3C1941:TDKAME%3E2.0.CO;2 + + + """ + # get raw bin counts + freq, height_edges, field_edges = np.histogram2d(altitude_data.compressed(), field_data.compressed(), + bins=[altitude_bins, field_bins]) + + # sum counts over y axis (height) + freq_sum = np.sum(freq, axis=1) + # get threshold for normalizing + point_thres = min_frac_thres * np.max(freq_sum) + # repeat to create array same size as freq + freq_sum_rep = np.repeat(freq_sum[..., np.newaxis], freq.shape[1], axis=1) + # normalize + freq_norm = freq / freq_sum_rep + # mask data where there is not enough points + freq_norm = np.ma.masked_where(freq_sum_rep < point_thres, freq_norm) + + return freq_norm, height_edges, field_edges From c9d6ce25651b78b62221eb659e65cee35852207c Mon Sep 17 00:00:00 2001 From: lauratomkins Date: Tue, 5 Dec 2023 14:47:52 -0500 Subject: [PATCH 02/18] Update __init__.py --- pyart/retrieve/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pyart/retrieve/__init__.py b/pyart/retrieve/__init__.py index 3b79ba5391..f74a5c9bfc 100644 --- a/pyart/retrieve/__init__.py +++ b/pyart/retrieve/__init__.py @@ -29,5 +29,6 @@ from .simple_moment_calculations import compute_snr # noqa from .spectra_calculations import dealias_spectra, spectra_moments # noqa from .vad import vad_browning, vad_michelson # noqa +from .cfad import create_cfad # noqa __all__ = [s for s in dir() if not s.startswith("_")] From 5998daebbe688f36157c4fa57fba03213465e296 Mon Sep 17 00:00:00 2001 From: lauratomkins Date: Tue, 5 Dec 2023 14:47:54 -0500 Subject: [PATCH 03/18] Create test_cfad.py --- tests/retrieve/test_cfad.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 tests/retrieve/test_cfad.py diff --git a/tests/retrieve/test_cfad.py b/tests/retrieve/test_cfad.py new file mode 100644 index 0000000000..73b3c2af44 --- /dev/null +++ b/tests/retrieve/test_cfad.py @@ -0,0 +1,31 @@ +""" Unit Tests for Py-ART's retrieve/echo_class.py module. """ + +import numpy as np + +import pyart + + +def test_cfad_default(): + + # set row to mask and test + verify_index = 5 + # create random grid of reflectivity data + ref_random_full = np.random.random((10, 10)) * 30 + # create a mask to mask 90% data from a specific altitude + mask = np.zeros_like(ref_random_full) + mask[verify_index, 1:] = 1 + # mask reflectivity data + ref_random_mask = np.ma.masked_where(mask, ref_random_full) + # create altitude data + z_col = np.linspace(0, 12000, 10) + z_full = np.repeat(z_col[..., np.newaxis], 10, axis=1) + z_mask = np.ma.masked_where(ref_random_mask.mask, z_full) + # compute CFAD + freq_norm, height_edges, field_edges = pyart.retrieve.create_cfad(ref_random_mask, + z_mask, + field_bins=np.linspace(0, 30, 20), + altitude_bins=np.linspace(0, 12000, 10) + ) + # if CFAD code works correctly, all values in this row should be false since this altitude has only 1 value (less + # than the necessary fraction needed) + assert freq_norm[verify_index, :].mask.all() From 2c6204bda6092cfb9cd19744b6088a1188084e7f Mon Sep 17 00:00:00 2001 From: lauratomkins Date: Tue, 5 Dec 2023 14:47:57 -0500 Subject: [PATCH 04/18] Create plot_cfad.py --- examples/retrieve/plot_cfad.py | 101 +++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 examples/retrieve/plot_cfad.py diff --git a/examples/retrieve/plot_cfad.py b/examples/retrieve/plot_cfad.py new file mode 100644 index 0000000000..33e8e5218d --- /dev/null +++ b/examples/retrieve/plot_cfad.py @@ -0,0 +1,101 @@ +""" +======================================= +Creating a CFAD diagram +======================================= +This example shows how to create a contoured frequency by altitude (CFAD) diagram +""" + +print(__doc__) + +# Author: Laura Tomkins (lmtomkin@ncsu.edu) +# License: BSD 3 clause + + +import matplotlib.pyplot as plt +import numpy as np + +import pyart + +###################################### +# Example 1 + +# get test data +filename = pyart.testing.get_test_data("sgpxsaprrhicmacI5.c0.20110524.015604_NC4.nc") +radar = pyart.io.read_cfradial(filename) + +# compute CFAD +# get reflectivity data and mask extremes +subset_slice = radar.get_slice(0) +ref_data = radar.fields['reflectivity_horizontal']['data'][subset_slice] +ref_data_masked = np.ma.masked_outside(ref_data, -15, 35) +# get altitude data +_, _, gate_z = radar.get_gate_x_y_z(0) +gate_z_masked = np.ma.masked_where(ref_data_masked.mask, gate_z) + +freq_norm, height_edges, field_edges = createCFAD(ref_data_masked, gate_z_masked, field_bins=np.linspace(-15,35,100), + altitude_bins=np.linspace(0,15000,100), min_frac_thres=0.1) + +# plot CFAD +freq_norm_masked = np.ma.masked_less_equal(freq_norm, 0) + +fig = plt.figure() +ax = plt.axes() +cfad_pm = ax.pcolormesh(field_edges, height_edges, freq_norm_masked, cmap='plasma', vmin=0, vmax=0.10) +plt.colorbar(cfad_pm) +ax.set_xlabel('Reflectivity [dBZ]') +ax.set_ylabel('Height [m]') +ax.grid(ls='--', color='gray', lw=0.5, alpha=0.7) +plt.show() + +# plot RHI data +display = pyart.graph.RadarDisplay(radar) + +fig = plt.figure(figsize=[12,3]) +ax = plt.axes() +plt.tight_layout() +display.plot('reflectivity_horizontal', 0, vmin=-15, vmax=35, mask_outside=True, cmap='magma_r', ax=ax) +display.set_limits(ylim=[0,15], ax=ax) +ax.set_aspect('equal') +plt.show() + +###################################### +# Example 2 + +# get test data +filename = pyart.testing.get_test_data("034142.mdv") +radar = pyart.io.read_mdv(filename) + +# compute CFAD +# get reflectivity data and mask extremes +subset_slice = radar.get_slice(0) +ref_data = radar.fields['reflectivity']['data'][subset_slice] +ref_data_masked = np.ma.masked_outside(ref_data, -5, 60) +# get altitude data +_, _, gate_z = radar.get_gate_x_y_z(0) +gate_z_masked = np.ma.masked_where(ref_data_masked.mask, gate_z) + +freq_norm, height_edges, field_edges = createCFAD(ref_data_masked, gate_z_masked, field_bins=np.linspace(-5,60,100), + altitude_bins=np.linspace(0,20000,100), min_frac_thres=0.1) + +# plot CFAD +freq_norm_masked = np.ma.masked_less_equal(freq_norm, 0) + +fig = plt.figure() +ax = plt.axes() +cfad_pm = ax.pcolormesh(field_edges, height_edges, freq_norm_masked, cmap='plasma', vmin=0, vmax=0.10) +plt.colorbar(cfad_pm) +ax.set_xlabel('Reflectivity [dBZ]') +ax.set_ylabel('Height [m]') +ax.grid(ls='--', color='gray', lw=0.5, alpha=0.7) +plt.show() + +# plot RHI data +display = pyart.graph.RadarDisplay(radar) + +fig = plt.figure(figsize=[12,3]) +ax = plt.axes() +plt.tight_layout() +display.plot('reflectivity', 0, vmin=-5, vmax=60, cmap='magma_r', ax=ax) +display.set_limits(ylim=[0,20], xlim=[0,80], ax=ax) +ax.set_aspect('equal') +plt.show() \ No newline at end of file From fd5ff953a7f64d3efc0566400e1c136258047345 Mon Sep 17 00:00:00 2001 From: lauratomkins Date: Tue, 5 Dec 2023 16:49:58 -0500 Subject: [PATCH 05/18] Update plot_cfad.py --- examples/retrieve/plot_cfad.py | 271 ++++++++++++++++++++++++++++----- 1 file changed, 233 insertions(+), 38 deletions(-) diff --git a/examples/retrieve/plot_cfad.py b/examples/retrieve/plot_cfad.py index 33e8e5218d..1cdfb650d2 100644 --- a/examples/retrieve/plot_cfad.py +++ b/examples/retrieve/plot_cfad.py @@ -13,11 +13,39 @@ import matplotlib.pyplot as plt import numpy as np +from open_radar_data import DATASETS import pyart + +###################################### +# Description of a CFAD +# ---------- +# A contoured frequency by altitude diagram (CFAD) is essentially a 2D histogram used to depict the vertical +# distribution of a particular variable, such as radar reflectivity. The x-axis represents the frequency of the +# field and the y-axis represents the frequency of the altitude. A key feature that distinguishes a CFAD from a +# regular 2D histogram is that it is normalized by altitude and altitudes where there is insufficient data are +# removed. See Yuter and Houze (1995) for full details of the diagram. +# +# **Interpretation** +# In a CFAD diagram, for a given altitude, the normalized frequency values will sum to a value of 1. When +# interpreting a CFAD diagram, the normalized frequency for a given bin will be the frequency of that value for the +# associated altitude (i.e. each bin can be interpreted as the fraction of data points at each altitude). +# +# **Minimum fraction threshold** +# In a CFAD diagram, altitudes where there is insufficient data are removed. This feature is controlled with the +# `min_frac_thres` value. The default value is 0.1, meaning that altitudes where the number of observations is less +# than one tenth of the maximum number of observations across all altitudes are removed. Increasing the value will +# act to be more aggressive in removing altitudes and decreasing the value will act to include more data. Setting +# this value to zero will include all altitudes with data, but use caution when interpreting CFADs with all data +# available as those altitudes with less data will not be as representative as other altitudes with sufficient data. ###################################### -# Example 1 +# Basic example with reflectivity from RHI +# ---------- +# **Example with RHI** +# First, we will show an example from an RHI scan. We will get the reflectivity data and mask outside -15 and 35 dBZ +# to remove any noisy data. For the best results, we recommend cleaning up the field as much as possible (i.e. +# despeckling, filtering, etc.) # get test data filename = pyart.testing.get_test_data("sgpxsaprrhicmacI5.c0.20110524.015604_NC4.nc") @@ -26,76 +54,243 @@ # compute CFAD # get reflectivity data and mask extremes subset_slice = radar.get_slice(0) -ref_data = radar.fields['reflectivity_horizontal']['data'][subset_slice] +ref_data = radar.fields["reflectivity_horizontal"]["data"][subset_slice] ref_data_masked = np.ma.masked_outside(ref_data, -15, 35) # get altitude data _, _, gate_z = radar.get_gate_x_y_z(0) gate_z_masked = np.ma.masked_where(ref_data_masked.mask, gate_z) -freq_norm, height_edges, field_edges = createCFAD(ref_data_masked, gate_z_masked, field_bins=np.linspace(-15,35,100), - altitude_bins=np.linspace(0,15000,100), min_frac_thres=0.1) +freq_norm, height_edges, field_edges = pyart.retrieve.create_cfad( + ref_data_masked, + gate_z_masked, + field_bins=np.linspace(-15, 35, 100), + altitude_bins=np.linspace(0, 15000, 50), + min_frac_thres=0.1, +) # plot CFAD freq_norm_masked = np.ma.masked_less_equal(freq_norm, 0) fig = plt.figure() ax = plt.axes() -cfad_pm = ax.pcolormesh(field_edges, height_edges, freq_norm_masked, cmap='plasma', vmin=0, vmax=0.10) +cfad_pm = ax.pcolormesh( + field_edges, height_edges/1000, freq_norm_masked, cmap="plasma", vmin=0, vmax=0.10 +) plt.colorbar(cfad_pm) -ax.set_xlabel('Reflectivity [dBZ]') -ax.set_ylabel('Height [m]') -ax.grid(ls='--', color='gray', lw=0.5, alpha=0.7) +ax.set_xlabel("Reflectivity [dBZ]") +ax.set_ylabel("Height [km]") +ax.grid(ls="--", color="gray", lw=0.5, alpha=0.7) plt.show() # plot RHI data display = pyart.graph.RadarDisplay(radar) -fig = plt.figure(figsize=[12,3]) +plt.figure(figsize=[12, 3]) ax = plt.axes() plt.tight_layout() -display.plot('reflectivity_horizontal', 0, vmin=-15, vmax=35, mask_outside=True, cmap='magma_r', ax=ax) -display.set_limits(ylim=[0,15], ax=ax) -ax.set_aspect('equal') +display.plot( + "reflectivity_horizontal", + 0, + vmin=-15, + vmax=35, + mask_outside=True, + cmap="magma_r", + ax=ax, +) +display.set_limits(ylim=[0, 15], ax=ax) +ax.set_aspect("equal") plt.show() ###################################### -# Example 2 +# Minimum fraction threshold example +# ---------- +# Previously, we used the default `min_frac_thres` of 0.1. Next, we will increase the threshold and set the threshold +# to 0 (not recommended) so show how the CFAD changes. -# get test data -filename = pyart.testing.get_test_data("034142.mdv") -radar = pyart.io.read_mdv(filename) +# Let's see the effect of changing the minimum fraction threshold: +freq_norm2, height_edges, field_edges = pyart.retrieve.create_cfad( + ref_data_masked, + gate_z_masked, + field_bins=np.linspace(-15, 35, 100), + altitude_bins=np.linspace(0, 15000, 50), + min_frac_thres=0.2, +) +freq_norm0, height_edges, field_edges = pyart.retrieve.create_cfad( + ref_data_masked, + gate_z_masked, + field_bins=np.linspace(-15, 35, 100), + altitude_bins=np.linspace(0, 15000, 50), + min_frac_thres=0, +) -# compute CFAD -# get reflectivity data and mask extremes -subset_slice = radar.get_slice(0) -ref_data = radar.fields['reflectivity']['data'][subset_slice] -ref_data_masked = np.ma.masked_outside(ref_data, -5, 60) -# get altitude data -_, _, gate_z = radar.get_gate_x_y_z(0) -gate_z_masked = np.ma.masked_where(ref_data_masked.mask, gate_z) +# plot CFAD +freq_norm2_masked = np.ma.masked_less_equal(freq_norm2, 0) +freq_norm0_masked = np.ma.masked_less_equal(freq_norm0, 0) + +plt.figure(figsize=(12,3)) +ax1 = plt.subplot(1, 3, 3) +cfad_pm = ax1.pcolormesh( + field_edges, height_edges/1000, freq_norm2_masked, cmap="plasma", vmin=0, vmax=0.10 +) +plt.colorbar(cfad_pm, ax=ax1) +ax1.set_xlabel("Reflectivity [dBZ]") +ax1.grid(ls="--", color="gray", lw=0.5, alpha=0.7) +ax1.set_title("min_frac_thres = 0.2") -freq_norm, height_edges, field_edges = createCFAD(ref_data_masked, gate_z_masked, field_bins=np.linspace(-5,60,100), - altitude_bins=np.linspace(0,20000,100), min_frac_thres=0.1) +ax2 = plt.subplot(1, 3, 2) +cfad_pm = ax2.pcolormesh( + field_edges, height_edges/1000, freq_norm_masked, cmap="plasma", vmin=0, vmax=0.10 +) +plt.colorbar(cfad_pm, ax=ax2) +ax2.set_xlabel("Reflectivity [dBZ]") +ax2.grid(ls="--", color="gray", lw=0.5, alpha=0.7) +ax2.set_title("min_frac_thres = 0.1") + +ax3 = plt.subplot(1, 3, 1) +cfad_pm = ax3.pcolormesh( + field_edges, height_edges/1000, freq_norm0_masked, cmap="plasma", vmin=0, vmax=0.10 +) +plt.colorbar(cfad_pm, ax=ax3) +ax3.set_xlabel("Reflectivity [dBZ]") +ax3.set_ylabel("Height [km]") +ax3.grid(ls="--", color="gray", lw=0.5, alpha=0.7) +ax3.set_title("min_frac_thres = 0") +plt.show() + +###################################### +# Setting the `min_frac_thres` to 0 (left panel) shows all data, even near the top of the chart (14km) where there is +# limited echo. Setting the `min_frac_thres` higher to 0.2 (right panel) removed altitudes between 1 and 5 km where +# there is less echo. +# +# +# Velocity example +# ---------- +# Next, we will show a CFAD for the doppler velocity from the above example. First, we have to dealias the velocity + +# create a gatefilter +gatefilter = pyart.filters.GateFilter(radar) +gatefilter.exclude_invalid("reflectivity_horizontal") +gatefilter.exclude_outside("reflectivity_horizontal", -15, 30) +gatefilter.exclude_invalid("mean_doppler_velocity") + +velocity_dealias = pyart.correct.dealias_region_based( + radar, + gatefilter=gatefilter, + vel_field="mean_doppler_velocity", + nyquist_vel=13, +) +radar.add_field("corrected_velocity", velocity_dealias) + +# plot RHI data +display = pyart.graph.RadarDisplay(radar) + +plt.figure(figsize=[12, 3]) +ax = plt.axes() +plt.tight_layout() +display.plot( + "corrected_velocity", + 0, + vmin=-20, + vmax=20, + mask_outside=True, + cmap="RdBu_r", + ax=ax, +) +display.set_limits(ylim=[0, 15], ax=ax) +ax.set_aspect("equal") +plt.show() + +# now do the CFAD +vel_data = radar.fields["corrected_velocity"]["data"][subset_slice] +vel_data_masked = np.ma.masked_where(gate_z_masked.mask, vel_data) +gate_z_masked = np.ma.masked_where(vel_data_masked.mask, gate_z_masked) + +freq_norm, height_edges, field_edges = pyart.retrieve.create_cfad( + vel_data_masked, + gate_z_masked, + field_bins=np.linspace(-30, 30, 50), + altitude_bins=np.linspace(0, 15000, 50), + min_frac_thres=0.2, +) # plot CFAD freq_norm_masked = np.ma.masked_less_equal(freq_norm, 0) fig = plt.figure() ax = plt.axes() -cfad_pm = ax.pcolormesh(field_edges, height_edges, freq_norm_masked, cmap='plasma', vmin=0, vmax=0.10) +cfad_pm = ax.pcolormesh( + field_edges, + height_edges/1000, + freq_norm_masked, + cmap="plasma", + vmin=0, + vmax=0.20, +) plt.colorbar(cfad_pm) -ax.set_xlabel('Reflectivity [dBZ]') -ax.set_ylabel('Height [m]') -ax.grid(ls='--', color='gray', lw=0.5, alpha=0.7) +ax.set_xlabel("Velocity [m s$^{-1}$]") +ax.set_ylabel("Height [km]") +ax.grid(ls="--", color="gray", lw=0.5, alpha=0.7) plt.show() -# plot RHI data -display = pyart.graph.RadarDisplay(radar) -fig = plt.figure(figsize=[12,3]) +###################################### +# Validation +# ---------- +# Finally, we wanted to compare this function with the original method, so here we recreate Fig. 2c from Yuter and +# Houze (1995) to demonstrate that it works the same. + +# get test data +filename = DATASETS.fetch("ddop.910815.213931.cdf") +grid = pyart.io.read_grid(filename) + +# get fields +altitude_data = grid.point_z["data"] +field_data = grid.fields["maxdz"]["data"][:] +vvel_data = grid.fields["w_wind"]["data"][:] + +# now mask data and correct altitude +vvel_masked = np.ma.masked_invalid(vvel_data) +field_data_masked = np.ma.masked_less_equal(field_data, -15) +field_data_masked = np.ma.masked_where(vvel_masked.mask, field_data_masked) +altitude_data_masked = np.ma.masked_where(field_data_masked.mask, altitude_data - 800) + +# define histogram bins +field_bins = np.arange(-20, 65, 5) +altitude_bins = np.arange(-0.2, 18.5, 0.4) + +freq_norm, height_edges, field_edges = pyart.retrieve.create_cfad( + field_data_masked, + altitude_data_masked / 1000, + field_bins=field_bins, + altitude_bins=altitude_bins, + min_frac_thres=0.1, +) + +# plot CFAD with contour plot +freq_norm_masked = np.ma.masked_less_equal(freq_norm, 0) +h, f = np.meshgrid(height_edges, field_edges) + +plt.figure(figsize=(6, 3)) ax = plt.axes() -plt.tight_layout() -display.plot('reflectivity', 0, vmin=-5, vmax=60, cmap='magma_r', ax=ax) -display.set_limits(ylim=[0,20], xlim=[0,80], ax=ax) -ax.set_aspect('equal') -plt.show() \ No newline at end of file +cont = ax.contour( + f[:-1, :-1] + 2.5, + h[:-1, :-1] + 0.2, + freq_norm.T, + levels=np.arange(0.05, 0.3, 0.05), + colors='black', +) +ax.set_yticks([0, 5, 10, 15]) +ax.set_xticks([-10, 0, 10, 20, 30, 40, 50]) +ax.set_xlim([-12, 58]) +ax.set_ylim([-0.5, 16]) +ax.set_ylabel("Height [km]") +ax.set_xlabel("Reflectivity [dBZ]") +ax.axhline(8, ls="--", lw=0.75, color="black") +ax.set_title("Recreation of Yuter and Houze (1995) Fig. 2c") +plt.show() + +# References +# ---------- +# Yuter, S. E., and R. A. Houze, 1995: Three-Dimensional Kinematic and Microphysical Evolution of Florida +# Cumulonimbus. Part II: Frequency Distributions of Vertical Velocity, Reflectivity, and Differential Reflectivity. +# Mon. Wea. Rev. 123, 1941-1963. https://doi.org/10.1175/1520-0493(1995)123%3C1941:TDKAME%3E2.0.CO;2 From e669e47e90420787c759884f5e01ee614e9893ac Mon Sep 17 00:00:00 2001 From: lauratomkins Date: Tue, 5 Dec 2023 16:50:03 -0500 Subject: [PATCH 06/18] Update plot_feature_detection.py --- examples/retrieve/plot_feature_detection.py | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/retrieve/plot_feature_detection.py b/examples/retrieve/plot_feature_detection.py index 5122ce6928..64de056996 100644 --- a/examples/retrieve/plot_feature_detection.py +++ b/examples/retrieve/plot_feature_detection.py @@ -398,7 +398,6 @@ ###################################### # Part 2: Cool-season feature detection # ---------- -###################################### # **Winter storm example** # # In this example, we will show how to algorithm can be used to detect features (snow bands) in winter storms. Here, we From d78610931750d836954278519330bbe241df091e Mon Sep 17 00:00:00 2001 From: lauratomkins Date: Tue, 5 Dec 2023 16:50:07 -0500 Subject: [PATCH 07/18] Update cfad.py --- pyart/retrieve/cfad.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pyart/retrieve/cfad.py b/pyart/retrieve/cfad.py index 4f13ed063d..e40e115ecf 100644 --- a/pyart/retrieve/cfad.py +++ b/pyart/retrieve/cfad.py @@ -49,8 +49,11 @@ def create_cfad( """ # get raw bin counts - freq, height_edges, field_edges = np.histogram2d(altitude_data.compressed(), field_data.compressed(), - bins=[altitude_bins, field_bins]) + freq, height_edges, field_edges = np.histogram2d( + altitude_data.compressed(), + field_data.compressed(), + bins=[altitude_bins, field_bins], + ) # sum counts over y axis (height) freq_sum = np.sum(freq, axis=1) From 237283bbab1815cdb401fdbfc0362fcf22941fab Mon Sep 17 00:00:00 2001 From: lauratomkins Date: Tue, 5 Dec 2023 16:50:10 -0500 Subject: [PATCH 08/18] Update test_cfad.py --- tests/retrieve/test_cfad.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/retrieve/test_cfad.py b/tests/retrieve/test_cfad.py index 73b3c2af44..b1e958ca39 100644 --- a/tests/retrieve/test_cfad.py +++ b/tests/retrieve/test_cfad.py @@ -21,11 +21,12 @@ def test_cfad_default(): z_full = np.repeat(z_col[..., np.newaxis], 10, axis=1) z_mask = np.ma.masked_where(ref_random_mask.mask, z_full) # compute CFAD - freq_norm, height_edges, field_edges = pyart.retrieve.create_cfad(ref_random_mask, - z_mask, - field_bins=np.linspace(0, 30, 20), - altitude_bins=np.linspace(0, 12000, 10) - ) + freq_norm, height_edges, field_edges = pyart.retrieve.create_cfad( + ref_random_mask, + z_mask, + field_bins=np.linspace(0, 30, 20), + altitude_bins=np.linspace(0, 12000, 10), + ) # if CFAD code works correctly, all values in this row should be false since this altitude has only 1 value (less # than the necessary fraction needed) assert freq_norm[verify_index, :].mask.all() From ca060f651eeb23d86d2d0afda9db0a806f646723 Mon Sep 17 00:00:00 2001 From: lauratomkins Date: Tue, 5 Dec 2023 17:03:31 -0500 Subject: [PATCH 09/18] Update plot_cfad.py --- examples/retrieve/plot_cfad.py | 61 +++++++++++++++++++++++++--------- 1 file changed, 46 insertions(+), 15 deletions(-) diff --git a/examples/retrieve/plot_cfad.py b/examples/retrieve/plot_cfad.py index 1cdfb650d2..ea51c93895 100644 --- a/examples/retrieve/plot_cfad.py +++ b/examples/retrieve/plot_cfad.py @@ -60,6 +60,7 @@ _, _, gate_z = radar.get_gate_x_y_z(0) gate_z_masked = np.ma.masked_where(ref_data_masked.mask, gate_z) +# get CFAD freq_norm, height_edges, field_edges = pyart.retrieve.create_cfad( ref_data_masked, gate_z_masked, @@ -69,12 +70,13 @@ ) # plot CFAD +# mask frequency values less than zero freq_norm_masked = np.ma.masked_less_equal(freq_norm, 0) - -fig = plt.figure() +# plot CFAD +plt.figure() ax = plt.axes() cfad_pm = ax.pcolormesh( - field_edges, height_edges/1000, freq_norm_masked, cmap="plasma", vmin=0, vmax=0.10 + field_edges, height_edges / 1000, freq_norm_masked, cmap="plasma", vmin=0, vmax=0.10 ) plt.colorbar(cfad_pm) ax.set_xlabel("Reflectivity [dBZ]") @@ -102,6 +104,11 @@ plt.show() ###################################### +# We can see a general increase in reflectivity values from the echo top to around 8km. The maximum frequency value in +# each altitude row represents the mode of the reflectivity distribution which we can see also increases from echo +# top to 8km. Below 8km the distribution of reflectivity values widens, likely associated with some of the noise in +# the RHI. + # Minimum fraction threshold example # ---------- # Previously, we used the default `min_frac_thres` of 0.1. Next, we will increase the threshold and set the threshold @@ -124,13 +131,19 @@ ) # plot CFAD +# mask zero values freq_norm2_masked = np.ma.masked_less_equal(freq_norm2, 0) freq_norm0_masked = np.ma.masked_less_equal(freq_norm0, 0) - -plt.figure(figsize=(12,3)) +# plot +plt.figure(figsize=(12, 3)) ax1 = plt.subplot(1, 3, 3) cfad_pm = ax1.pcolormesh( - field_edges, height_edges/1000, freq_norm2_masked, cmap="plasma", vmin=0, vmax=0.10 + field_edges, + height_edges / 1000, + freq_norm2_masked, + cmap="plasma", + vmin=0, + vmax=0.10, ) plt.colorbar(cfad_pm, ax=ax1) ax1.set_xlabel("Reflectivity [dBZ]") @@ -139,7 +152,12 @@ ax2 = plt.subplot(1, 3, 2) cfad_pm = ax2.pcolormesh( - field_edges, height_edges/1000, freq_norm_masked, cmap="plasma", vmin=0, vmax=0.10 + field_edges, + height_edges / 1000, + freq_norm_masked, + cmap="plasma", + vmin=0, + vmax=0.10, ) plt.colorbar(cfad_pm, ax=ax2) ax2.set_xlabel("Reflectivity [dBZ]") @@ -148,7 +166,12 @@ ax3 = plt.subplot(1, 3, 1) cfad_pm = ax3.pcolormesh( - field_edges, height_edges/1000, freq_norm0_masked, cmap="plasma", vmin=0, vmax=0.10 + field_edges, + height_edges / 1000, + freq_norm0_masked, + cmap="plasma", + vmin=0, + vmax=0.10, ) plt.colorbar(cfad_pm, ax=ax3) ax3.set_xlabel("Reflectivity [dBZ]") @@ -160,12 +183,13 @@ ###################################### # Setting the `min_frac_thres` to 0 (left panel) shows all data, even near the top of the chart (14km) where there is # limited echo. Setting the `min_frac_thres` higher to 0.2 (right panel) removed altitudes between 1 and 5 km where -# there is less echo. +# there is less echo than between 6 and 12km where there is a consistent swath of reflectivity throughout the entire +# cross section. # # # Velocity example # ---------- -# Next, we will show a CFAD for the doppler velocity from the above example. First, we have to dealias the velocity +# Next, we will show a CFAD for the doppler velocity from the above example. First, we have to dealias the velocity. # create a gatefilter gatefilter = pyart.filters.GateFilter(radar) @@ -173,6 +197,7 @@ gatefilter.exclude_outside("reflectivity_horizontal", -15, 30) gatefilter.exclude_invalid("mean_doppler_velocity") +# dealias velocity velocity_dealias = pyart.correct.dealias_region_based( radar, gatefilter=gatefilter, @@ -214,13 +239,15 @@ ) # plot CFAD +# mask zero values freq_norm_masked = np.ma.masked_less_equal(freq_norm, 0) -fig = plt.figure() +# plot +plt.figure() ax = plt.axes() cfad_pm = ax.pcolormesh( field_edges, - height_edges/1000, + height_edges / 1000, freq_norm_masked, cmap="plasma", vmin=0, @@ -232,12 +259,16 @@ ax.grid(ls="--", color="gray", lw=0.5, alpha=0.7) plt.show() - ###################################### +# The velocity CFAD is very different from the reflectivity CFAD. In most altitudes, there is more of a bimodal +# pattern associated with the changing sign of the velocity values on either side of the radar. In general, +# the distribution of velocity values is consistently wide throughout the profile compared to the reflectivity CFAD. +# # Validation # ---------- # Finally, we wanted to compare this function with the original method, so here we recreate Fig. 2c from Yuter and -# Houze (1995) to demonstrate that it works the same. +# Houze (1995) to demonstrate that it works the same. Instead of using the `pcolormesh` function, we are using +# contour lines. # get test data filename = DATASETS.fetch("ddop.910815.213931.cdf") @@ -277,7 +308,7 @@ h[:-1, :-1] + 0.2, freq_norm.T, levels=np.arange(0.05, 0.3, 0.05), - colors='black', + colors="black", ) ax.set_yticks([0, 5, 10, 15]) ax.set_xticks([-10, 0, 10, 20, 30, 40, 50]) From 4fc4e15d72c7aa17ba5efe228b7d0b7d195d3de9 Mon Sep 17 00:00:00 2001 From: lauratomkins Date: Tue, 5 Dec 2023 17:05:43 -0500 Subject: [PATCH 10/18] linting updates --- examples/retrieve/plot_cfad.py | 3 ++- tests/retrieve/test_cfad.py | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/retrieve/plot_cfad.py b/examples/retrieve/plot_cfad.py index ea51c93895..8226865c6d 100644 --- a/examples/retrieve/plot_cfad.py +++ b/examples/retrieve/plot_cfad.py @@ -268,7 +268,8 @@ # ---------- # Finally, we wanted to compare this function with the original method, so here we recreate Fig. 2c from Yuter and # Houze (1995) to demonstrate that it works the same. Instead of using the `pcolormesh` function, we are using -# contour lines. +# contour lines. + # get test data filename = DATASETS.fetch("ddop.910815.213931.cdf") diff --git a/tests/retrieve/test_cfad.py b/tests/retrieve/test_cfad.py index b1e958ca39..51cf79aee1 100644 --- a/tests/retrieve/test_cfad.py +++ b/tests/retrieve/test_cfad.py @@ -6,7 +6,6 @@ def test_cfad_default(): - # set row to mask and test verify_index = 5 # create random grid of reflectivity data From 82d9ec4d9692db1a10fc6b2443f75415af60c886 Mon Sep 17 00:00:00 2001 From: lauratomkins Date: Tue, 5 Dec 2023 17:06:48 -0500 Subject: [PATCH 11/18] linting update --- examples/retrieve/plot_cfad.py | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/retrieve/plot_cfad.py b/examples/retrieve/plot_cfad.py index 8226865c6d..8ad99d930c 100644 --- a/examples/retrieve/plot_cfad.py +++ b/examples/retrieve/plot_cfad.py @@ -17,7 +17,6 @@ import pyart - ###################################### # Description of a CFAD # ---------- From ee1afca1826f9c2dc51c39c2292a3b6bda47b33d Mon Sep 17 00:00:00 2001 From: lauratomkins Date: Tue, 23 Jan 2024 15:18:47 -0500 Subject: [PATCH 12/18] Update to use grid or radar object --- pyart/retrieve/cfad.py | 38 +++++++++++++++++++++++++++++++------- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/pyart/retrieve/cfad.py b/pyart/retrieve/cfad.py index e40e115ecf..28d00ccb66 100644 --- a/pyart/retrieve/cfad.py +++ b/pyart/retrieve/cfad.py @@ -7,10 +7,11 @@ def create_cfad( - field_data, - altitude_data, + radar, field_bins, altitude_bins, + field="reflectivity", + field_mask=None, min_frac_thres=0.1, ): """ @@ -18,15 +19,17 @@ def create_cfad( histogram that is normalized by the number of points at each altitude. Altitude bins are masked where the counts are less than a minimum fraction of the largest number of counts for any altitude row. - field_data : array - Array of radar data to use for CFAD calculation. - altitude_data : array - Array of corresponding altitude data to use for CFAD calculation. - Note: must be the same shape as `field_data` + radar : Radar + Radar object used. Can be Radar or Grid object. field_bins : list List of bin edges for field values to use for CFAD creation. altitude_bins : list List of bin edges for height values to use for CFAD creation. + field : str + Field name to use to look up reflectivity data. In the + radar object. Default field name is 'reflectivity'. + field_mask : array + An array the same size as the field array used to mask values. min_frac_thres : float, optional Fraction of values to remove in CFAD normalization (default 0.1). If an altitude row has a total count that is less than min_frac_thres of the largest number of total counts for any altitude row, the bins in that @@ -48,6 +51,27 @@ def create_cfad( """ + + # get field data + field_data = radar.fields[field]["data"][:] + + # get altitude data + # first try to get altitude data from a radar object + try: + altitude_data = radar.gate_z['data'] + # if it fails, try to get altitude data from a grid object + except: + try: + altitude_data = radar.point_z["data"] + finally: + print("No altitude data found") + raise + + # option to mask data if a mask is given + if field_mask is not None: + field_data = np.ma.masked_where(field_mask, field_data) + altitude_data = np.ma.masked_where(field_data.mask, altitude_data) + # get raw bin counts freq, height_edges, field_edges = np.histogram2d( altitude_data.compressed(), From 7ff9020baf7d8674179be7436137a7a5acfb528c Mon Sep 17 00:00:00 2001 From: lauratomkins Date: Tue, 23 Jan 2024 15:44:09 -0500 Subject: [PATCH 13/18] corrections --- pyart/retrieve/cfad.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyart/retrieve/cfad.py b/pyart/retrieve/cfad.py index 28d00ccb66..cc93bd4770 100644 --- a/pyart/retrieve/cfad.py +++ b/pyart/retrieve/cfad.py @@ -58,12 +58,12 @@ def create_cfad( # get altitude data # first try to get altitude data from a radar object try: - altitude_data = radar.gate_z['data'] + altitude_data = radar.gate_z["data"] # if it fails, try to get altitude data from a grid object except: try: altitude_data = radar.point_z["data"] - finally: + except: print("No altitude data found") raise From 057cc7eb6a067a59e7c48ee8bcd43edd5d7a21d1 Mon Sep 17 00:00:00 2001 From: lauratomkins Date: Tue, 23 Jan 2024 15:53:18 -0500 Subject: [PATCH 14/18] update example --- examples/retrieve/plot_cfad.py | 52 +++++++++++++++++----------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/examples/retrieve/plot_cfad.py b/examples/retrieve/plot_cfad.py index 8ad99d930c..cffeba98ff 100644 --- a/examples/retrieve/plot_cfad.py +++ b/examples/retrieve/plot_cfad.py @@ -52,19 +52,20 @@ # compute CFAD # get reflectivity data and mask extremes -subset_slice = radar.get_slice(0) -ref_data = radar.fields["reflectivity_horizontal"]["data"][subset_slice] +# extract first sweep +radar = radar.extract_sweeps([0]) +# get mask array +ref_data = radar.fields["reflectivity_horizontal"]["data"][:] ref_data_masked = np.ma.masked_outside(ref_data, -15, 35) -# get altitude data -_, _, gate_z = radar.get_gate_x_y_z(0) -gate_z_masked = np.ma.masked_where(ref_data_masked.mask, gate_z) +field_mask = ref_data_masked.mask # get CFAD freq_norm, height_edges, field_edges = pyart.retrieve.create_cfad( - ref_data_masked, - gate_z_masked, + radar, field_bins=np.linspace(-15, 35, 100), altitude_bins=np.linspace(0, 15000, 50), + field="reflectivity_horizontal", + field_mask=field_mask, min_frac_thres=0.1, ) @@ -115,17 +116,19 @@ # Let's see the effect of changing the minimum fraction threshold: freq_norm2, height_edges, field_edges = pyart.retrieve.create_cfad( - ref_data_masked, - gate_z_masked, + radar, field_bins=np.linspace(-15, 35, 100), altitude_bins=np.linspace(0, 15000, 50), + field="reflectivity_horizontal", + field_mask=field_mask, min_frac_thres=0.2, ) freq_norm0, height_edges, field_edges = pyart.retrieve.create_cfad( - ref_data_masked, - gate_z_masked, + radar, field_bins=np.linspace(-15, 35, 100), altitude_bins=np.linspace(0, 15000, 50), + field="reflectivity_horizontal", + field_mask=field_mask, min_frac_thres=0, ) @@ -224,16 +227,12 @@ ax.set_aspect("equal") plt.show() -# now do the CFAD -vel_data = radar.fields["corrected_velocity"]["data"][subset_slice] -vel_data_masked = np.ma.masked_where(gate_z_masked.mask, vel_data) -gate_z_masked = np.ma.masked_where(vel_data_masked.mask, gate_z_masked) - freq_norm, height_edges, field_edges = pyart.retrieve.create_cfad( - vel_data_masked, - gate_z_masked, + radar, field_bins=np.linspace(-30, 30, 50), altitude_bins=np.linspace(0, 15000, 50), + field="corrected_velocity", + field_mask=field_mask, min_frac_thres=0.2, ) @@ -265,7 +264,7 @@ # # Validation # ---------- -# Finally, we wanted to compare this function with the original method, so here we recreate Fig. 2c from Yuter and +# Finally, we wanted to compare this function with the original method, so here we reproduce Fig. 2c from Yuter and # Houze (1995) to demonstrate that it works the same. Instead of using the `pcolormesh` function, we are using # contour lines. @@ -274,26 +273,27 @@ filename = DATASETS.fetch("ddop.910815.213931.cdf") grid = pyart.io.read_grid(filename) -# get fields +# make corrections to altitude field altitude_data = grid.point_z["data"] +grid.point_z["data"] = (altitude_data - 800) / 1000 + +# get fields to create a mask field_data = grid.fields["maxdz"]["data"][:] vvel_data = grid.fields["w_wind"]["data"][:] - -# now mask data and correct altitude vvel_masked = np.ma.masked_invalid(vvel_data) field_data_masked = np.ma.masked_less_equal(field_data, -15) field_data_masked = np.ma.masked_where(vvel_masked.mask, field_data_masked) -altitude_data_masked = np.ma.masked_where(field_data_masked.mask, altitude_data - 800) # define histogram bins field_bins = np.arange(-20, 65, 5) altitude_bins = np.arange(-0.2, 18.5, 0.4) freq_norm, height_edges, field_edges = pyart.retrieve.create_cfad( - field_data_masked, - altitude_data_masked / 1000, + grid, field_bins=field_bins, altitude_bins=altitude_bins, + field="maxdz", + field_mask=field_data_masked.mask, min_frac_thres=0.1, ) @@ -317,7 +317,7 @@ ax.set_ylabel("Height [km]") ax.set_xlabel("Reflectivity [dBZ]") ax.axhline(8, ls="--", lw=0.75, color="black") -ax.set_title("Recreation of Yuter and Houze (1995) Fig. 2c") +ax.set_title("Yuter and Houze (1995) Fig. 2c") plt.show() # References From 0f149da44a1a14d1cb734b15929b0096010a917c Mon Sep 17 00:00:00 2001 From: lauratomkins Date: Wed, 24 Jan 2024 10:10:53 -0500 Subject: [PATCH 15/18] Update test_cfad.py --- tests/retrieve/test_cfad.py | 44 ++++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/tests/retrieve/test_cfad.py b/tests/retrieve/test_cfad.py index 51cf79aee1..ecf1eca8a2 100644 --- a/tests/retrieve/test_cfad.py +++ b/tests/retrieve/test_cfad.py @@ -6,26 +6,30 @@ def test_cfad_default(): - # set row to mask and test - verify_index = 5 - # create random grid of reflectivity data - ref_random_full = np.random.random((10, 10)) * 30 - # create a mask to mask 90% data from a specific altitude - mask = np.zeros_like(ref_random_full) - mask[verify_index, 1:] = 1 - # mask reflectivity data - ref_random_mask = np.ma.masked_where(mask, ref_random_full) - # create altitude data - z_col = np.linspace(0, 12000, 10) - z_full = np.repeat(z_col[..., np.newaxis], 10, axis=1) - z_mask = np.ma.masked_where(ref_random_mask.mask, z_full) - # compute CFAD + # initalize test radar object + radar = pyart.io.read(pyart.testing.NEXRAD_ARCHIVE_MSG31_FILE) + ref_field = "reflectivity" + + # set every value to 20 + radar.fields[ref_field]["data"] = np.ones(radar.fields[ref_field]["data"].shape) * 20 + + # set mask to none + field_mask = np.zeros(radar.fields[ref_field]["data"].shape) + + # calculate CFAD freq_norm, height_edges, field_edges = pyart.retrieve.create_cfad( - ref_random_mask, - z_mask, + radar, field_bins=np.linspace(0, 30, 20), - altitude_bins=np.linspace(0, 12000, 10), + altitude_bins=np.arange(0, 18000, 100), + field="reflectivity", + field_mask=field_mask, ) - # if CFAD code works correctly, all values in this row should be false since this altitude has only 1 value (less - # than the necessary fraction needed) - assert freq_norm[verify_index, :].mask.all() + + # set row to mask and test + verify_index = 12 + + # if CFAD code works correctly, each column should have the same values and only 1 column should have a value of 1 + # check all columns are the same + assert freq_norm.all(axis=0).any() + # check column 12 is all ones + assert (freq_norm[:, verify_index] == 1).all() From a90935030bcf862d3d2bf772d326254792624096 Mon Sep 17 00:00:00 2001 From: lauratomkins Date: Wed, 24 Jan 2024 10:12:47 -0500 Subject: [PATCH 16/18] Update test_cfad.py --- tests/retrieve/test_cfad.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/retrieve/test_cfad.py b/tests/retrieve/test_cfad.py index ecf1eca8a2..fceaeb4dfb 100644 --- a/tests/retrieve/test_cfad.py +++ b/tests/retrieve/test_cfad.py @@ -11,7 +11,9 @@ def test_cfad_default(): ref_field = "reflectivity" # set every value to 20 - radar.fields[ref_field]["data"] = np.ones(radar.fields[ref_field]["data"].shape) * 20 + radar.fields[ref_field]["data"] = ( + np.ones(radar.fields[ref_field]["data"].shape) * 20 + ) # set mask to none field_mask = np.zeros(radar.fields[ref_field]["data"].shape) From 9a27f24e08af2b1790bdd9d6f3fac1abf99ade6c Mon Sep 17 00:00:00 2001 From: lauratomkins Date: Wed, 24 Jan 2024 10:13:56 -0500 Subject: [PATCH 17/18] Update test_cfad.py --- tests/retrieve/test_cfad.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/retrieve/test_cfad.py b/tests/retrieve/test_cfad.py index fceaeb4dfb..aab688c9f4 100644 --- a/tests/retrieve/test_cfad.py +++ b/tests/retrieve/test_cfad.py @@ -12,7 +12,7 @@ def test_cfad_default(): # set every value to 20 radar.fields[ref_field]["data"] = ( - np.ones(radar.fields[ref_field]["data"].shape) * 20 + np.ones(radar.fields[ref_field]["data"].shape) * 20 ) # set mask to none From 0c7aba1cbcbf04d917692e5673e1476c19c39794 Mon Sep 17 00:00:00 2001 From: lauratomkins Date: Thu, 25 Jan 2024 15:02:43 -0500 Subject: [PATCH 18/18] add spaces --- pyart/retrieve/cfad.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyart/retrieve/cfad.py b/pyart/retrieve/cfad.py index cc93bd4770..a08c13246d 100644 --- a/pyart/retrieve/cfad.py +++ b/pyart/retrieve/cfad.py @@ -34,6 +34,7 @@ def create_cfad( Fraction of values to remove in CFAD normalization (default 0.1). If an altitude row has a total count that is less than min_frac_thres of the largest number of total counts for any altitude row, the bins in that altitude row are masked. + Returns ------- freq_norm : array @@ -42,6 +43,7 @@ def create_cfad( Array of bin edges for height data. field_edges : array of x coordinates Array of bin edges for field data. + References ---------- Yuter, S. E., and R. A. Houze, 1995: Three-Dimensional Kinematic and