Increase testcoverage on github (#1110)

* replace westernscheldt dataset, including update of testcase and notebook

* added fou and rst to data.py including tests

* made test_zsigmalayermodel_correct_layers work on github

* made fou testcase work on github

* made test_rename_waqvars work on github

* made test_open_2Dnetwork_with_1Dtopology work on github

* made test_enrich_rst_with_map work on github

* made test_uds_auto_set_crs_spherical work on github

* removed test_interpolate_nc_to_bc since all called functions are also covered by other tests

dfm_tools/data.py

@@ -4,7 +4,10 @@
 import xugrid as xu
 import pooch
 import zipfile
-from dfm_tools.xugrid_helpers import open_partitioned_dataset, open_dataset_delft3d4
+from dfm_tools.xugrid_helpers import (open_partitioned_dataset,
+                                      open_dataset_delft3d4,
+                                      enrich_rst_with_map,
+                                      )
 from dfm_tools.xarray_helpers import preprocess_hisnc
 
 __all__ = ["fm_grevelingen_map",
@@ -144,12 +147,14 @@ def fm_curvedbend_his(return_filepath:bool = False) -> xr.Dataset:
 
 
 def fm_westernscheldt_map(return_filepath:bool = False) -> xu.UgridDataset:
+    # from p:\dflowfm\maintenance\JIRA\05000-05999\05477\c103_ws_3d_fourier
 
+    dir_subfolder = 'DFM_westernscheldt_3D'
     dir_testdata = get_dir_testdata()
 
     #download data if not present
-    file_nc = os.path.join(dir_testdata,'westernscheldt_sph_map.nc')
-    maybe_download_opendap_data(file_nc)
+    file_nc = os.path.join(dir_testdata,'westerscheldt01_0subst_map.nc')
+    maybe_download_opendap_data(file_nc,dir_subfolder)
 
     #potentially only return filepath of downloaded file(s)
     filepath = file_nc
@@ -161,6 +166,66 @@ def fm_westernscheldt_map(return_filepath:bool = False) -> xu.UgridDataset:
     return uds
 
 
+def fm_westernscheldt_fou(return_filepath:bool = False) -> xu.UgridDataset:
+    # from p:\dflowfm\maintenance\JIRA\05000-05999\05477\c103_ws_3d_fourier
+
+    dir_subfolder = 'DFM_westernscheldt_3D'
+    dir_testdata = get_dir_testdata()
+
+    #download data if not present
+    file_nc = os.path.join(dir_testdata,'westerscheldt01_0subst_fou.nc')
+    maybe_download_opendap_data(file_nc,dir_subfolder)
+
+    #potentially only return filepath of downloaded file(s)
+    filepath = file_nc
+    if return_filepath:
+        return filepath
+
+    #open as UgridDataset
+    uds = open_partitioned_dataset(filepath, remove_edges=True)
+    return uds
+
+
+def fm_westernscheldt_rst(return_filepath:bool = False) -> xu.UgridDataset:
+    # from p:\dflowfm\maintenance\JIRA\05000-05999\05477\c103_ws_3d_fourier
+
+    dir_subfolder = 'DFM_westernscheldt_3D'
+    dir_testdata = get_dir_testdata()
+
+    #download data if not present
+    file_nc = os.path.join(dir_testdata,'westerscheldt01_0subst_20140101_004640_rst.nc')
+    maybe_download_opendap_data(file_nc,dir_subfolder)
+
+    #potentially only return filepath of downloaded file(s)
+    filepath = file_nc
+    if return_filepath:
+        return filepath
+
+    #open as UgridDataset
+    uds = open_partitioned_dataset(filepath, preprocess=enrich_rst_with_map)
+    return uds
+
+
+def fm_westernscheldt_his(return_filepath:bool = False) -> xr.Dataset:
+    # from p:\dflowfm\maintenance\JIRA\05000-05999\05477\c103_ws_3d_fourier
+
+    dir_subfolder = 'DFM_westernscheldt_3D'
+    dir_testdata = get_dir_testdata()
+
+    #download data if not present
+    file_nc = os.path.join(dir_testdata,'westerscheldt01_0subst_his.nc')
+    maybe_download_opendap_data(file_nc,dir_subfolder)
+
+    #potentially only return filepath of downloaded file(s)
+    filepath = file_nc
+    if return_filepath:
+        return filepath
+
+    #open as xarray.Dataset
+    ds = xr.open_mfdataset(filepath,preprocess=preprocess_hisnc)
+    return ds
+
+
 def d3d_westernscheldt_trim(return_filepath:bool = False) -> xu.UgridDataset:
 
     dir_testdata = get_dir_testdata()
@@ -247,6 +312,8 @@ def download_gshhs(url):
         file_url = f'https://www.ngdc.noaa.gov/mgg/shorelines/data/gshhg/latest/{fname}.zip'
         resp = download_gshhs(file_url)
         if resp.is_redirect:
+            # get zipfile from USHLC in case of NOAA server maintenance
+            # https://github.com/Deltares/dfm_tools/issues/1111
             print("failed (redirected), trying different source")
             file_url = f'https://www.soest.hawaii.edu/pwessel/gshhg/{fname}.zip'
             resp = download_gshhs(file_url)

tests/test_data.py

@@ -17,6 +17,8 @@ def test_data_map():
                  dfmt.data.fm_grevelingen_net, 
                  dfmt.data.fm_curvedbend_map, 
                  dfmt.data.fm_westernscheldt_map, 
+                 dfmt.data.fm_westernscheldt_fou, 
+                 dfmt.data.fm_westernscheldt_rst, 
                  dfmt.data.d3d_westernscheldt_trim,
                  dfmt.data.d3d_curvedbend_trim]
 
@@ -31,6 +33,7 @@ def test_data_map():
 def test_data_his():
     func_list = [dfmt.data.fm_grevelingen_his, 
                  dfmt.data.fm_curvedbend_his,
+                 dfmt.data.fm_westernscheldt_his,
                  dfmt.data.d3d_curvedbend_trih]
 
     for func in func_list:

tests/test_get_nc.py

@@ -25,36 +25,36 @@ def test_zlayermodel_correct_layers():
     """
 
     file_nc = dfmt.data.fm_grevelingen_map(return_filepath=True) #zlayer
-    data_frommap_merged = dfmt.open_partitioned_dataset(file_nc)
+    uds = dfmt.open_partitioned_dataset(file_nc)
 
     timestep = 3
 
-    data_frommap_timesel = data_frommap_merged.isel(time=timestep) #select data for all layers
-    data_frommap_merged_fullgrid = reconstruct_zw_zcc_fromz(data_frommap_timesel)
+    uds_timesel = uds.isel(time=timestep) #select data for all layers
+    uds_fullgrid = reconstruct_zw_zcc_fromz(uds_timesel)
 
-    vals_wl = data_frommap_merged_fullgrid['mesh2d_s1']
-    vals_bl = data_frommap_merged_fullgrid['mesh2d_flowelem_bl'].to_numpy()
+    vals_wl = uds_fullgrid['mesh2d_s1']
+    vals_bl = uds_fullgrid['mesh2d_flowelem_bl'].to_numpy()
 
-    vals_zw_max = data_frommap_merged_fullgrid['mesh2d_flowelem_zw'].max(dim='nmesh2d_interface')
-    vals_zw_min = data_frommap_merged_fullgrid['mesh2d_flowelem_zw'].min(dim='nmesh2d_interface')
+    vals_zw_max = uds_fullgrid['mesh2d_flowelem_zw'].max(dim='nmesh2d_interface')
+    vals_zw_min = uds_fullgrid['mesh2d_flowelem_zw'].min(dim='nmesh2d_interface')
     assert np.allclose(vals_zw_max, vals_wl)
     assert np.allclose(vals_zw_min, vals_bl)
 
     # check z-centers in one specific cell
-    zcc = data_frommap_merged_fullgrid['mesh2d_flowelem_zcc']
+    zcc = uds_fullgrid['mesh2d_flowelem_zcc']
     zcc_onecell = zcc.isel(nmesh2d_face=5000).load().fillna(-999)
     zcc_onecell_expected = np.array([-999]*32 + [ -4.1403    , -3.375     , -2.125, -0.84786265])
     assert np.allclose(zcc_onecell, zcc_onecell_expected)
 
     # check z-interfaces in one specific cell
-    zw = data_frommap_merged_fullgrid['mesh2d_flowelem_zw']
+    zw = uds_fullgrid['mesh2d_flowelem_zw']
     zw_onecell = zw.isel(nmesh2d_face=5000).load().fillna(-999)
     zw_onecell_expected = np.array([-999]*32 + [-4.2806, -4.0, -2.75, -1.5, -0.195725292])
     assert np.allclose(zw_onecell, zw_onecell_expected)
 
     # check if all non-dry centers are below waterlevel and above bed
-    vals_zcc_max = data_frommap_merged_fullgrid['mesh2d_flowelem_zcc'].max(dim='nmesh2d_layer').to_numpy()
-    vals_zcc_min = data_frommap_merged_fullgrid['mesh2d_flowelem_zcc'].min(dim='nmesh2d_layer').to_numpy()
+    vals_zcc_max = uds_fullgrid['mesh2d_flowelem_zcc'].max(dim='nmesh2d_layer').to_numpy()
+    vals_zcc_min = uds_fullgrid['mesh2d_flowelem_zcc'].min(dim='nmesh2d_layer').to_numpy()
     bool_dry = vals_wl == vals_bl
     assert ((vals_zcc_max < vals_wl) | bool_dry).all()
     assert ((vals_zcc_min > vals_bl) | bool_dry).all()
@@ -90,47 +90,47 @@ def test_zlayermodel_correct_layers_nanabovewl():
     assert np.allclose(zw_onecell, zw_onecell_expected)
 
 
-@pytest.mark.requireslocaldata
 @pytest.mark.unittest
 def test_zsigmalayermodel_correct_layers():
     """
     we assert top/max/min only, since zlayers can also be valid when the bottom layer contains nan values
     """
 
-    file_nc = r'p:\dflowfm\maintenance\JIRA\05000-05999\05477\c103_ws_3d_fourier\DFM_OUTPUT_westerscheldt01_0subst\westerscheldt01_0subst_map.nc' #zsigma model without fullgrid output but with new ocean_sigma_z_coordinate variable
-    data_frommap_merged = dfmt.open_partitioned_dataset(file_nc)
+    # zsigma model without fullgrid output but with new ocean_sigma_z_coordinate variable
+    file_nc = dfmt.data.fm_westernscheldt_map(return_filepath=True)
+    uds = dfmt.open_partitioned_dataset(file_nc)
 
     timestep = 1
 
-    data_frommap_timesel = data_frommap_merged.isel(time=timestep) #select data for all layers
-    data_frommap_merged_fullgrid = reconstruct_zw_zcc_fromzsigma(data_frommap_timesel)
+    uds_timesel = uds.isel(time=timestep) #select data for all layers
+    uds_fullgrid = reconstruct_zw_zcc_fromzsigma(uds_timesel)
 
-    vals_wl = data_frommap_merged_fullgrid['mesh2d_s1'].to_numpy()
-    vals_bl = data_frommap_merged_fullgrid['mesh2d_flowelem_bl'].to_numpy()
+    vals_wl = uds_fullgrid['mesh2d_s1'].to_numpy()
+    vals_bl = uds_fullgrid['mesh2d_flowelem_bl'].to_numpy()
 
     # check z-centers in one specific cell
-    zcc = data_frommap_merged_fullgrid['mesh2d_flowelem_zcc']
+    zcc = uds_fullgrid['mesh2d_flowelem_zcc']
     zcc_onecell = zcc.isel(mesh2d_nFaces=5000).load().fillna(-999)
     zcc_onecell_expected = np.array([-999, -999, -999, -4.86792313, -0.250909869])
     assert np.allclose(zcc_onecell, zcc_onecell_expected)
 
     # check z-interfaces in one specific cell
-    zw = data_frommap_merged_fullgrid['mesh2d_flowelem_zw']
+    zw = uds_fullgrid['mesh2d_flowelem_zw']
     zw_onecell = zw.isel(mesh2d_nFaces=5000).load().fillna(-999)
     zw_onecell_expected = np.array([-999., -999., -999.,   -7.17642976, -2.5594165, 2.05759676])
     assert np.allclose(zw_onecell, zw_onecell_expected)
 
-    vals_zw_max = data_frommap_merged_fullgrid['mesh2d_flowelem_zw'].max(dim='mesh2d_nInterfaces').to_numpy()
-    vals_zw_min = data_frommap_merged_fullgrid['mesh2d_flowelem_zw'].min(dim='mesh2d_nInterfaces').to_numpy()
-    vals_zw_top = data_frommap_merged_fullgrid['mesh2d_flowelem_zw'].isel(mesh2d_nInterfaces=-1).to_numpy()
+    vals_zw_max = uds_fullgrid['mesh2d_flowelem_zw'].max(dim='mesh2d_nInterfaces').to_numpy()
+    vals_zw_min = uds_fullgrid['mesh2d_flowelem_zw'].min(dim='mesh2d_nInterfaces').to_numpy()
+    vals_zw_top = uds_fullgrid['mesh2d_flowelem_zw'].isel(mesh2d_nInterfaces=-1).to_numpy()
     assert np.allclose(vals_zw_max, vals_wl)
     assert np.allclose(vals_zw_min, vals_bl)
     assert np.allclose(vals_zw_max, vals_zw_top)
 
     # check if all non-dry cell centers are below waterlevel and above bed
-    vals_zcc_max = data_frommap_merged_fullgrid['mesh2d_flowelem_zcc'].max(dim='mesh2d_nLayers').to_numpy()
-    vals_zcc_min = data_frommap_merged_fullgrid['mesh2d_flowelem_zcc'].min(dim='mesh2d_nLayers').to_numpy()
-    vals_zcc_top = data_frommap_merged_fullgrid['mesh2d_flowelem_zcc'].isel(mesh2d_nLayers=-1).to_numpy()
+    vals_zcc_max = uds_fullgrid['mesh2d_flowelem_zcc'].max(dim='mesh2d_nLayers').to_numpy()
+    vals_zcc_min = uds_fullgrid['mesh2d_flowelem_zcc'].min(dim='mesh2d_nLayers').to_numpy()
+    vals_zcc_top = uds_fullgrid['mesh2d_flowelem_zcc'].isel(mesh2d_nLayers=-1).to_numpy()
     bool_dry = vals_wl == vals_bl
     assert ((vals_zcc_max < vals_wl) | bool_dry).all()
     assert ((vals_zcc_min > vals_bl) | bool_dry).all()
@@ -148,24 +148,24 @@ def test_sigmalayermodel_correct_layers():
     timestep = 3
 
     data_frommap_timesel = data_frommap_merged.isel(time=timestep) #select data for all layers
-    data_frommap_merged_fullgrid = reconstruct_zw_zcc_fromsigma(data_frommap_timesel)
+    uds_fullgrid = reconstruct_zw_zcc_fromsigma(data_frommap_timesel)
 
-    vals_wl = data_frommap_merged_fullgrid['mesh2d_s1'].to_numpy()
-    vals_bl = data_frommap_merged_fullgrid['mesh2d_flowelem_bl'].to_numpy()
+    vals_wl = uds_fullgrid['mesh2d_s1'].to_numpy()
+    vals_bl = uds_fullgrid['mesh2d_flowelem_bl'].to_numpy()
 
-    vals_zw_max = data_frommap_merged_fullgrid['mesh2d_flowelem_zw'].max(dim='mesh2d_nInterfaces').to_numpy()
-    vals_zw_min = data_frommap_merged_fullgrid['mesh2d_flowelem_zw'].min(dim='mesh2d_nInterfaces').to_numpy()
-    vals_zw_top = data_frommap_merged_fullgrid['mesh2d_flowelem_zw'].isel(mesh2d_nInterfaces=-1).to_numpy()
-    vals_zw_bot = data_frommap_merged_fullgrid['mesh2d_flowelem_zw'].isel(mesh2d_nInterfaces=0).to_numpy()
+    vals_zw_max = uds_fullgrid['mesh2d_flowelem_zw'].max(dim='mesh2d_nInterfaces').to_numpy()
+    vals_zw_min = uds_fullgrid['mesh2d_flowelem_zw'].min(dim='mesh2d_nInterfaces').to_numpy()
+    vals_zw_top = uds_fullgrid['mesh2d_flowelem_zw'].isel(mesh2d_nInterfaces=-1).to_numpy()
+    vals_zw_bot = uds_fullgrid['mesh2d_flowelem_zw'].isel(mesh2d_nInterfaces=0).to_numpy()
     assert (np.abs(vals_zw_max-vals_wl)<1e-6).all()
     assert (np.abs(vals_zw_min-vals_bl)<1e-6).all()
     assert (np.abs(vals_zw_max-vals_zw_top)<1e-6).all()
     assert (np.abs(vals_zw_min-vals_zw_bot)<1e-6).all()
 
-    vals_zcc_max = data_frommap_merged_fullgrid['mesh2d_flowelem_zcc'].max(dim='mesh2d_nLayers').to_numpy()
-    vals_zcc_min = data_frommap_merged_fullgrid['mesh2d_flowelem_zcc'].min(dim='mesh2d_nLayers').to_numpy()
-    vals_zcc_top = data_frommap_merged_fullgrid['mesh2d_flowelem_zcc'].isel(mesh2d_nLayers=-1).to_numpy()
-    vals_zcc_bot = data_frommap_merged_fullgrid['mesh2d_flowelem_zcc'].isel(mesh2d_nLayers=0).to_numpy()
+    vals_zcc_max = uds_fullgrid['mesh2d_flowelem_zcc'].max(dim='mesh2d_nLayers').to_numpy()
+    vals_zcc_min = uds_fullgrid['mesh2d_flowelem_zcc'].min(dim='mesh2d_nLayers').to_numpy()
+    vals_zcc_top = uds_fullgrid['mesh2d_flowelem_zcc'].isel(mesh2d_nLayers=-1).to_numpy()
+    vals_zcc_bot = uds_fullgrid['mesh2d_flowelem_zcc'].isel(mesh2d_nLayers=0).to_numpy()
     assert (vals_zcc_max < vals_wl).all() # < works since there are no bl>0 in this model
     assert (vals_zcc_min > vals_bl).all()
     assert (np.abs(vals_zcc_max-vals_zcc_top)<1e-6).all()

tests/test_get_nc_helpers.py

@@ -26,24 +26,38 @@ def test_get_ncvarproperties():
     assert "projected_coordinate_system" in vars_pd.index
 
 
-@pytest.mark.requireslocaldata
 @pytest.mark.unittest
 def test_rename_waqvars():
-    file_nc = os.path.join(r'p:\archivedprojects\11203850-coastserv\06-Model\waq_model\simulations\run0_20200319\DFM_OUTPUT_kzn_waq', 'kzn_waq_0000_map.nc')
+    # originally tested with r'p:\archivedprojects\11203850-coastserv\06-Model\waq_model\simulations\run0_20200319\DFM_OUTPUT_kzn_waq\kzn_waq_0000_map.nc'
+    # but converted to a portable testcase by setting up a dummy dataset
+    file_nc = dfmt.data.fm_curvedbend_map(return_filepath=True)
     uds = dfmt.open_partitioned_dataset(file_nc)
+
+    # add dummy waq variable
+    waq_attrs = {'mesh': 'mesh2d',
+     'location': 'face',
+     'cell_methods': 'mesh2d_nFaces: mean',
+     'long_name': 'Chlfa',
+     'units': '(mg/m3)',
+     'grid_mapping': 'wgs84',
+     'description': 'Chlfa - Chlorophyll-a concentration (mg/m3) in flow element'}
+    uds['mesh2d_water_quality_output_61'] = uds['mesh2d_flowelem_bl'].assign_attrs(waq_attrs)
+    uds['mesh2d_water_quality_output_62'] = uds['mesh2d_flowelem_bl'].assign_attrs(waq_attrs)
+
     uds = dfmt.rename_waqvars(uds)
     assert 'mesh2d_Chlfa' in uds.data_vars
+    # in case of duplicates, only the first one is renamed
+    assert 'mesh2d_water_quality_output_62' in uds.data_vars
 
 
-@pytest.mark.requireslocaldata
 @pytest.mark.unittest
 def test_rename_fouvars_regular():
-    file_nc_fou = r'p:\archivedprojects\11203379-005-mwra-updated-bem\03_model\02_final\A72_ntsu0_kzlb2\DFM_OUTPUT_MB_02_fou\MB_02_0000_fou.nc'
+    file_nc_fou = dfmt.data.fm_westernscheldt_fou(return_filepath=True)
     uds = dfmt.open_partitioned_dataset(file_nc_fou)
     uds_renamed = dfmt.rename_fouvars(uds)
 
-    assert 'mesh2d_tem_mean_20160201000000_20160301000000' in uds_renamed.data_vars
-    assert 'mesh2d_uy_mean_20160101000000_20170101000000' in uds_renamed.data_vars
+    assert 'mesh2d_ux_mean_20140101000000_20140101000600' in uds_renamed.data_vars
+    assert 'mesh2d_uy_mean_20140101000000_20140101000600' in uds_renamed.data_vars
 
 
 @pytest.mark.requireslocaldata

tests/test_interpolate_grid2bnd.py

@@ -174,40 +174,6 @@ def test_plipointsDataset_fews_accepted():
     assert forcing0.quantityunitpair[1].quantity == 'salinitybnd'
 
 
-@pytest.mark.systemtest
-@pytest.mark.requireslocaldata
-def test_interpolate_nc_to_bc():
-    file_pli = r'p:\archivedprojects\11208054-004-dcsm-fm\models\model_input\bnd_cond\pli\DCSM-FM_OB_all_20181108.pli'
-    npoints = 3
-
-    # read polyfile as geodataframe
-    polyfile_object = hcdfm.PolyFile(file_pli)
-    gdf_points_all = dfmt.PolyFile_to_geodataframe_points(polyfile_object)
-    gdf_points = gdf_points_all.iloc[:npoints]
-
-    tstart = '2012-12-16 12:00'
-    tstop = '2013-01-01 12:00'
-
-    ncvarname = 'so'
-    dir_pattern = os.path.join(r'p:\1204257-dcsmzuno\data\CMEMS\nc\DCSM_allAvailableTimes',f'{ncvarname}_2012-12*.nc')
-
-    #open regulargridDataset and do some basic stuff (time selection, renaming depth/lat/lon/varname, converting units, etc)
-    data_xr_vars = open_prepare_dataset(dir_pattern=dir_pattern, quantity='salinitybnd', tstart=tstart, tstop=tstop)
-    #interpolate regulargridDataset to plipointsDataset
-    data_interp = dfmt.interp_regularnc_to_plipointsDataset(data_xr_reg=data_xr_vars, gdf_points=gdf_points)
-
-    #convert plipointsDataset to hydrolib ForcingModel
-    forcingmodel_object = dfmt.plipointsDataset_to_ForcingModel(plipointsDataset=data_interp)
-
-    forcing0 = forcingmodel_object.forcing[0]
-    assert isinstance(forcingmodel_object, hcdfm.ForcingModel)
-    assert isinstance(forcing0, hcdfm.T3D)
-    assert forcing0.quantityunitpair[1].unit == '1e-3'
-
-    # test whether so was renamed to salinitybnd
-    assert forcing0.quantityunitpair[1].quantity == 'salinitybnd'
-
-
 @pytest.mark.systemtest
 def test_plipointsDataset_to_ForcingModel_drop_allnan_points():
     #construct polyfile gdf
@@ -232,6 +198,7 @@ def test_plipointsDataset_to_ForcingModel_drop_allnan_points():
              3.2138258e-05],
             [2.4448847e-05, 2.6705173e-05, 2.6929094e-05, 2.5548252e-05,
              2.5827681e-05]]]
+
     ds = xr.Dataset()
     ds['longitude'] = xr.DataArray([-72.  , -71.75, -71.5 , -71.25, -71.  ], dims='longitude')
     ds['latitude'] = xr.DataArray([12.  , 12.25, 12.5 , 12.75, 13.  , 13.25], dims='latitude')