From bb491a5450b27fc3bdf65133a7dce5e18fff740d Mon Sep 17 00:00:00 2001 From: Leila Belabbassi Date: Tue, 15 Nov 2022 12:58:33 -0600 Subject: [PATCH 01/12] Update cfutil.py related to issue#950 [https://github.com/ioos/compliance-checker/issues/950] updated the function get_grid_mapping_variables to parse the attribute variables when it contains more than one string. --- compliance_checker/cfutil.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/compliance_checker/cfutil.py b/compliance_checker/cfutil.py index 7ace15fff..189331b03 100644 --- a/compliance_checker/cfutil.py +++ b/compliance_checker/cfutil.py @@ -778,7 +778,9 @@ def get_grid_mapping_variables(ds): grid_mapping_variables = set() for ncvar in ds.get_variables_by_attributes(grid_mapping=lambda x: x is not None): if ncvar.grid_mapping in ds.variables: - grid_mapping_variables.add(ncvar.grid_mapping) + if ' ' in ncvar.grid_mapping: + grid_mapping_list = [x.split(':')[0] for x in ncvar.grid_mapping.split(' ') if ':' in x] + grid_mapping_variables.add(ncvar.grid_mapping) return grid_mapping_variables From 9c0b9ca1fc2f757bca98a57f299db29d81ab17e2 Mon Sep 17 00:00:00 2001 From: Leila Belabbassi Date: Tue, 15 Nov 2022 13:04:41 -0600 Subject: [PATCH 02/12] Update test_cf.py related to issue # 950 [https://github.com/ioos/compliance-checker/issues/950] Mocked a new .nc file to test the grid_mapping requirements and recommendations --- compliance_checker/tests/test_cf.py | 51 ++++++++++++++++++++++++----- 1 file changed, 43 insertions(+), 8 deletions(-) diff --git a/compliance_checker/tests/test_cf.py b/compliance_checker/tests/test_cf.py index e3e3ac609..dff978324 100644 --- a/compliance_checker/tests/test_cf.py +++ b/compliance_checker/tests/test_cf.py @@ -1294,15 +1294,50 @@ def test_check_reduced_horizontal_grid(self): assert all(r.name == "§5.3 Reduced Horizontal Grid" for r in results) def test_check_grid_mapping(self): - dataset = self.load_dataset(STATIC_FILES["mapping"]) - results = self.cf.check_grid_mapping(dataset) +# dataset = self.load_dataset(STATIC_FILES["mapping"]) +# results = self.cf.check_grid_mapping(dataset) + +# assert len(results) == 6 +# assert len([r.value for r in results.values() if r.value[0] < r.value[1]]) == 0 +# expected_name = ( +# "§5.6 Horizontal Coordinate Reference Systems, Grid Mappings, Projections" +# ) +# assert all(r.name == expected_name for r in results.values()) + + # create Cells with grid_apping variable + dataset = MockTimeSeries() + + dataset.createVariable("temp", "d", ("time")) + dataset.createVariable("crsOSGB", "d") + dataset.createVariable("crsWGS84", "d") + + temp = dataset.variables["temp"] + temp.standard_name = "air_temperature" + temp.units = "K" + temp.coordinates = "lat lon" + temp.grid_mapping = "crsOSGB: time crsWGS84: lat lon" + + # create grid_mapping crsOSGB ; + crsOSGB = dataset.variables["crsOSGB"] + crsOSGB.grid_mapping_name = "transverse_mercator" + crsOSGB.semi_major_axis = 6377563.396 + crsOSGB.inverse_flattening = 299.3249646 + crsOSGB.ongitude_of_prime_meridian = 0.0 + crsOSGB.latitude_of_projection_origin = 49.0 + crsOSGB.longitude_of_central_meridian = -2.0 + crsOSGB.scale_factor_at_central_meridian = 0.9996012717 + crsOSGB.false_easting = 400000.0 + crsOSGB.false_northing = -100000.0 + crsOSGB.unit = "metre" + + # create grid_mapping crsWGS84 + crsWGS84 = dataset.variables["crsWGS84"] + crsWGS84.grid_mapping_name = "latitude_longitude" + crsWGS84.longitude_of_prime_meridian = 0.0 + crsWGS84.semi_major_axis = 6378137.0 + crsWGS84.inverse_flattening = 298.257223563 - assert len(results) == 6 - assert len([r.value for r in results.values() if r.value[0] < r.value[1]]) == 0 - expected_name = ( - "§5.6 Horizontal Coordinate Reference Systems, Grid Mappings, Projections" - ) - assert all(r.name == expected_name for r in results.values()) + results = self.cf.check_grid_mapping(dataset) def test_is_geophysical(self): From f1d05a408ed7811780c8f7129af539e45d58d2f6 Mon Sep 17 00:00:00 2001 From: Leila Belabbassi Date: Tue, 15 Nov 2022 13:33:26 -0600 Subject: [PATCH 03/12] Update cfutil.py fixed the for loop of the get_grid_mapping_variables function --- compliance_checker/cfutil.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/compliance_checker/cfutil.py b/compliance_checker/cfutil.py index 189331b03..f7c28b420 100644 --- a/compliance_checker/cfutil.py +++ b/compliance_checker/cfutil.py @@ -776,11 +776,12 @@ def get_grid_mapping_variables(ds): :param netCDF4.Dataset ds: An open netCDF4 Dataset """ grid_mapping_variables = set() - for ncvar in ds.get_variables_by_attributes(grid_mapping=lambda x: x is not None): - if ncvar.grid_mapping in ds.variables: - if ' ' in ncvar.grid_mapping: - grid_mapping_list = [x.split(':')[0] for x in ncvar.grid_mapping.split(' ') if ':' in x] - grid_mapping_variables.add(ncvar.grid_mapping) + for ncvar in ds.get_variables_by_attributes(grid_mapping=lambda x: x is not None): + if ' ' in ncvar.grid_mapping: + grid_mapping_list = [x.split(':')[0] for x in ncvar.grid_mapping.split(' ') if ':' in x] + for item in grid_mapping_list: + if item in ds.variables: + grid_mapping_variables.add(item) return grid_mapping_variables From 250c3d65c26be09cca16bd0f9432785835dc16fa Mon Sep 17 00:00:00 2001 From: Leila Belabbassi Date: Tue, 15 Nov 2022 17:11:41 -0600 Subject: [PATCH 04/12] Update test_cf.py --- compliance_checker/tests/test_cf.py | 83 ++++++++++++++++------------- 1 file changed, 47 insertions(+), 36 deletions(-) diff --git a/compliance_checker/tests/test_cf.py b/compliance_checker/tests/test_cf.py index dff978324..e3b569563 100644 --- a/compliance_checker/tests/test_cf.py +++ b/compliance_checker/tests/test_cf.py @@ -1293,7 +1293,7 @@ def test_check_reduced_horizontal_grid(self): assert len([r for r in results if r.value[0] < r.value[1]]) == 1 assert all(r.name == "§5.3 Reduced Horizontal Grid" for r in results) - def test_check_grid_mapping(self): +# def test_check_grid_mapping(self): # dataset = self.load_dataset(STATIC_FILES["mapping"]) # results = self.cf.check_grid_mapping(dataset) @@ -1303,41 +1303,6 @@ def test_check_grid_mapping(self): # "§5.6 Horizontal Coordinate Reference Systems, Grid Mappings, Projections" # ) # assert all(r.name == expected_name for r in results.values()) - - # create Cells with grid_apping variable - dataset = MockTimeSeries() - - dataset.createVariable("temp", "d", ("time")) - dataset.createVariable("crsOSGB", "d") - dataset.createVariable("crsWGS84", "d") - - temp = dataset.variables["temp"] - temp.standard_name = "air_temperature" - temp.units = "K" - temp.coordinates = "lat lon" - temp.grid_mapping = "crsOSGB: time crsWGS84: lat lon" - - # create grid_mapping crsOSGB ; - crsOSGB = dataset.variables["crsOSGB"] - crsOSGB.grid_mapping_name = "transverse_mercator" - crsOSGB.semi_major_axis = 6377563.396 - crsOSGB.inverse_flattening = 299.3249646 - crsOSGB.ongitude_of_prime_meridian = 0.0 - crsOSGB.latitude_of_projection_origin = 49.0 - crsOSGB.longitude_of_central_meridian = -2.0 - crsOSGB.scale_factor_at_central_meridian = 0.9996012717 - crsOSGB.false_easting = 400000.0 - crsOSGB.false_northing = -100000.0 - crsOSGB.unit = "metre" - - # create grid_mapping crsWGS84 - crsWGS84 = dataset.variables["crsWGS84"] - crsWGS84.grid_mapping_name = "latitude_longitude" - crsWGS84.longitude_of_prime_meridian = 0.0 - crsWGS84.semi_major_axis = 6378137.0 - crsWGS84.inverse_flattening = 298.257223563 - - results = self.cf.check_grid_mapping(dataset) def test_is_geophysical(self): @@ -1980,6 +1945,52 @@ def test_cell_measures(self): message = "Cell measure variable box_area referred to by PS is not present in dataset variables" assert message in messages + def test_check_grid_mapping(self): + # dataset = self.load_dataset(STATIC_FILES["mapping"]) + # results = self.cf.check_grid_mapping(dataset) + + # assert len(results) == 6 + # assert len([r.value for r in results.values() if r.value[0] < r.value[1]]) == 0 + # expected_name = ( + # "§5.6 Horizontal Coordinate Reference Systems, Grid Mappings, Projections" + # ) + # assert all(r.name == expected_name for r in results.values()) + + # create Cells with grid_apping variable + dataset = MockTimeSeries() + + dataset.createVariable("temp", "d", ("time")) + dataset.createVariable("crsOSGB", "d") + dataset.createVariable("crsWGS84", "d") + + temp = dataset.variables["temp"] + temp.standard_name = "air_temperature" + temp.units = "K" + temp.coordinates = "lat lon" + temp.grid_mapping = "crsOSGB: time crsWGS84: lat lon" + + # create grid_mapping crsOSGB ; + crsOSGB = dataset.variables["crsOSGB"] + crsOSGB.grid_mapping_name = "transverse_mercator" + crsOSGB.semi_major_axis = 6377563.396 + crsOSGB.inverse_flattening = 299.3249646 + crsOSGB.ongitude_of_prime_meridian = 0.0 + crsOSGB.latitude_of_projection_origin = 49.0 + crsOSGB.longitude_of_central_meridian = -2.0 + crsOSGB.scale_factor_at_central_meridian = 0.9996012717 + crsOSGB.false_easting = 400000.0 + crsOSGB.false_northing = -100000.0 + crsOSGB.unit = "metre" + + # create grid_mapping crsWGS84 + crsWGS84 = dataset.variables["crsWGS84"] + crsWGS84.grid_mapping_name = "latitude_longitude" + crsWGS84.longitude_of_prime_meridian = 0.0 + crsWGS84.semi_major_axis = 6378137.0 + crsWGS84.inverse_flattening = 298.257223563 + + results = self.cf.check_grid_mapping(dataset) + def test_variable_features(self): with MockTimeSeries() as dataset: # I hope to never see an attribute value like this, but since From 169511061a000cf1556908aa313930cfbb332be8 Mon Sep 17 00:00:00 2001 From: Leila Belabbassi Date: Fri, 9 Dec 2022 16:34:59 -0600 Subject: [PATCH 05/12] modification to 5.6 Grid Mappings and Projections #950 in cf_1_7.py - added "geographic_crs_name" to the function def _check_gmattr_existence_condition_ell_pmerid_hdatum(self, var): - Modified function def check_grid_mapping(self, ds): in cfutil.py - modified the function def grid_mapping_variables(ds) - added the function def get_grid_mapping_variables_coordinates(ds): in test_cf.py - modified the function def test_check_grid_mapping(self): --- compliance_checker/cf/cf_1_7.py | 153 +++++++++++++++++++++++++--- compliance_checker/cfutil.py | 35 ++++++- compliance_checker/tests/test_cf.py | 92 +++++++++++------ 3 files changed, 233 insertions(+), 47 deletions(-) diff --git a/compliance_checker/cf/cf_1_7.py b/compliance_checker/cf/cf_1_7.py index 0729dacb0..85ea6a5d9 100644 --- a/compliance_checker/cf/cf_1_7.py +++ b/compliance_checker/cf/cf_1_7.py @@ -237,9 +237,12 @@ def check_cell_boundaries(self, ds): # Note that test does not check monotonicity ret_val = [] reasoning = [] + for variable_name, boundary_variable_name in cfutil.get_cell_boundary_map( ds ).items(): + + print(boundary_variable_name, boundary_variable_name.dtype) variable = ds.variables[variable_name] valid = True reasoning = [] @@ -479,7 +482,7 @@ def _check_gmattr_existence_condition_geoid_name_geoptl_datum_name(self, var): def _check_gmattr_existence_condition_ell_pmerid_hdatum(self, var): """ If one of reference_ellipsoid_name, prime_meridian_name, or - horizontal_datum_name are defined as grid_mapping attributes, + horizontal_datum_name, geographic_crs_name are defined as grid_mapping attributes, they must all be defined. :param netCDF4.Variable var @@ -501,6 +504,7 @@ def _check_gmattr_existence_condition_ell_pmerid_hdatum(self, var): "reference_ellipsoid_name", "prime_meridian_name", "horizontal_datum_name", + "geographic_crs_name", ] ] ) and ( @@ -509,6 +513,7 @@ def _check_gmattr_existence_condition_ell_pmerid_hdatum(self, var): "reference_ellipsoid_name", "prime_meridian_name", "horizontal_datum_name", + "geographic_crs_name", ] ).issubset(_ncattrs) ): @@ -724,17 +729,107 @@ def _evaluate_towgs84(self, val): def check_grid_mapping(self, ds): super(CF1_7Check, self).check_grid_mapping.__doc__ prev_return = super(CF1_7Check, self).check_grid_mapping(ds) - grid_mapping_variables = cfutil.get_grid_mapping_variables(ds) - for var_name in sorted(grid_mapping_variables): + test_ctx = self.get_test_ctx( + BaseCheck.HIGH, self.section_titles["5.6"]) + # Get the grid_mapping variable list for Requirements 5.6 [.../9] + grid_mapping_variable_list = [] + + for item in ds.get_variables_by_attributes(grid_mapping=lambda x: x is not None): + # [1/9] The type of the grid_mapping attribute is a string whose value + # is of the following form, in which brackets indicate optional + # text: grid_mapping_name[: coord_var [coord_var ...]][grid_mapping_name: [coord_var ... ]] + if not isinstance(item.grid_mapping, str): + test_ctx.messages.append("The grid_mapping attribute {} " + "is not a string".format(item.grid_mapping)) + test_ctx.out_of += 1 + if ' ' in item.grid_mapping: + for x in item.grid_mapping.replace(":", '').split(): + grid_mapping_variable_list.append(x) + else: + # [2/9] Note that in its simplest form the attribute comprises just + # a grid_mapping_name as a single word. + grid_mapping_variable_list.append(item.grid_mapping) + # + # Get the CF grid_mapping variables list (== only if the variable exist in the Variables list) + grid_mapping_variables = cfutil.get_grid_mapping_variables(ds) + grid_mapping_variables_coordinates = cfutil.get_grid_mapping_variables_coordinates(ds) + + #Check if all grid_mapping variables are listed in the file as expected + # [3/9] Each grid_mapping_name is the name of a variable (known as a + # grid mapping variable), which must exist in the file. + if len(grid_mapping_variable_list) != len(grid_mapping_variables) + len(grid_mapping_variables_coordinates): + test_ctx.messages.append("Not all of the grid_mapping variables exist in the file") + test_ctx.out_of += 1 + + test_ctx.score += 1 + + # Get the CF coordinate variables or auxiliary coordinate variables + auxiliary_coordinate_variables = cfutil.get_auxiliary_coordinate_variables(ds) + coordinate_variables = cfutil.get_coordinate_variables(ds) + # Check the grid_mapping variable coordinates if listed. + # [4/9] Each coord_var is the name of a coordinate variable or + # auxiliary coordinate variable, which must exist in the file. If it + # is an auxiliary coordinate variable, it must be listed in the + # coordinates attribute. + for coord_item in grid_mapping_variables_coordinates: + if coord_item not in coordinate_variables and coord_item not in auxiliary_coordinate_variables: + test_ctx.messages.append("{} is not listed as coordinate or auxiliary coord Variable.".format(coord_item)) + test_ctx.out_of += 1 + + test_ctx.score += 1 + + # Check the gid_mapping variable attributes + for var_name in sorted(grid_mapping_variables): var = ds.variables[var_name] test_ctx = self.get_test_ctx( - BaseCheck.HIGH, self.section_titles["5.6"], var.name - ) + BaseCheck.HIGH, self.section_titles["5.6"], var.name) + + # Recommendation: The grid mapping variables should have 0 dimensions. + if var.dimensions: + test_ctx.messages.append("The grid mapping variable {} should have 0 dimension".format(var.name)) + test_ctx.out_of += 1 + + test_ctx.score += 1 + + # [5/9] The grid mapping variables must have the grid_mapping_name attribute. + # The legal values for the grid_mapping_name attribute are contained in Appendix F. + if "grid_mapping_name" not in var.ncattrs(): + test_ctx.messages.append("The grid mapping variable " + "{} doesn't have the grid_mapping_name attribute".format(var.name)) + test_ctx.out_of += 1 + else: + if var.grid_mapping_name not in grid_mapping_dict17: + test_ctx.messages.append("The legal values for the grid_mapping_name attribute " + "{} is not contained in Appendix F".format(var.grid_mapping_name)) + test_ctx.out_of += 1 + + test_ctx.score += 1 + + # [6/9] The data types of the attributes of the + # grid mapping variable must be specified in Table 1 of Appendix F. + type_map = {"S": "str" , "N": "float64"} + for attrs_name in var.ncattrs(): + if attrs_name in grid_mapping_attr_types17: + attrs_type = np.dtype(type(getattr(var,attrs_name))) + attrs_type17 = type_map[grid_mapping_attr_types17[attrs_name]['type']] + if attrs_type17 != attrs_type: + test_ctx.messages.append("The data types of the attributes " + "{} and {} do not match".format(attrs_type17, attrs_type)) + test_ctx.out_of += 1 + else: + test_ctx.messages.append("The attribute {} " + "does not exist in Table 1 of Appendix F".format(attrs_name)) + test_ctx.out_of += 1 + test_ctx.score += 1 + # TODO: check cases where crs_wkt provides part of a necessary # grid_mapping attribute, or where a grid_mapping attribute # overrides what has been provided in crs_wkt. # attempt to parse crs_wkt if it is present + + # [7/9] If present, the crs_wkt attribute must be a text string conforming to + # the CRS WKT specification described in reference [OGC_CTS]. if "crs_wkt" in var.ncattrs(): crs_wkt = var.crs_wkt if not isinstance(crs_wkt, str): @@ -749,23 +844,39 @@ def check_grid_mapping(self, ds): str(crs_error) ) ) - else: - test_ctx.score += 1 - test_ctx.out_of += 1 - - # existence_conditions + test_ctx.out_of += 1 + test_ctx.score += 1 + + # existence_conditions exist_cond_1 = ( self._check_gmattr_existence_condition_geoid_name_geoptl_datum_name(var) ) - test_ctx.assert_true(exist_cond_1[0], exist_cond_1[1]) + if exist_cond_1[0] == False: + test_ctx.messages.append("Both geoid_name and geopotential_datum_name " + "should not exist for {}".format(var.name)) + test_ctx.out_of += 1 + + test_ctx.score += 1 + # test_ctx.assert_true(exist_cond_1[0], exist_cond_1[1]) + + # [8/9] reference_ellipsoid_name, prime_meridian_name, horizontal_datum_name and + # geographic_crs_name must be all defined if any one is defined. exist_cond_2 = self._check_gmattr_existence_condition_ell_pmerid_hdatum(var) - test_ctx.assert_true(exist_cond_2[0], exist_cond_2[1]) + if exist_cond_2[0] == False: + test_ctx.messages.append("reference_ellipsoid_name, prime_meridian_name, " + "horizontal_datum_name and geographic_crs_name must be all defined " + "if any one is defined") + test_ctx.out_of += 1 + + test_ctx.score += 1 + # test_ctx.assert_true(exist_cond_2[0], exist_cond_2[1]) # handle vertical datum related grid_mapping attributes vert_datum_attrs = {} possible_vert_datum_attrs = {"geoid_name", "geopotential_datum_name"} vert_datum_attrs = possible_vert_datum_attrs.intersection(var.ncattrs()) - len_vdatum_name_attrs = len(vert_datum_attrs) + len_vdatum_name_attrs = len(vert_datum_attrs) + # check that geoid_name and geopotential_datum_name are not both # present in the grid_mapping variable if len_vdatum_name_attrs == 2: @@ -800,8 +911,22 @@ def check_grid_mapping(self, ds): "Error occurred while trying to query " "Proj4 SQLite database at {}: {}".format(proj_db_path, str(e)) ) - prev_return[var.name] = test_ctx.to_result() + # [9/9] Check If projected_crs_name is defined then geographic_crs_name must be also. + test_possible = {"projected_crs_name","geographic_crs_name"} + if "projected_crs_name" in var.ncattrs(): + test_attr = test_possible.intersection(var.ncattrs()) + len_test_attr = len(test_attr) + + if len_test_attr != 2: + test_ctx.messages.append("We do not have both projected_crs_name " + "and geographic_crs_name defined") + test_ctx.out_of += 1 + + test_ctx.score += 1 + + prev_return[var.name] = test_ctx.to_result() + return prev_return def check_standard_name_deprecated_modifiers(self, ds): diff --git a/compliance_checker/cfutil.py b/compliance_checker/cfutil.py index f7c28b420..7ed5a863f 100644 --- a/compliance_checker/cfutil.py +++ b/compliance_checker/cfutil.py @@ -357,6 +357,7 @@ def get_cell_boundary_map(ds): for variable in ds.get_variables_by_attributes(bounds=lambda x: x is not None): if variable.bounds in ds.variables: boundary_map[variable.name] = variable.bounds + return boundary_map @@ -776,14 +777,42 @@ def get_grid_mapping_variables(ds): :param netCDF4.Dataset ds: An open netCDF4 Dataset """ grid_mapping_variables = set() - for ncvar in ds.get_variables_by_attributes(grid_mapping=lambda x: x is not None): - if ' ' in ncvar.grid_mapping: - grid_mapping_list = [x.split(':')[0] for x in ncvar.grid_mapping.split(' ') if ':' in x] + for ncvar in ds.get_variables_by_attributes(grid_mapping=lambda x: x is not None and isinstance(x, str)): + + if ' ' in ncvar.grid_mapping: + grid_mapping_list = [ + x.split(':')[0] for x in ncvar.grid_mapping.split(' ') if ':' in x + ] + if grid_mapping_list: for item in grid_mapping_list: if item in ds.variables: grid_mapping_variables.add(item) + else: + if ncvar.grid_mapping in ds.variables: + grid_mapping_variables.add(ncvar.grid_mapping) + return grid_mapping_variables +def get_grid_mapping_variables_coordinates(ds): + """ + Returns a list of grid mapping variables' coordinates + + :param netCDF4.Dataset ds: An open netCDF4 Dataset + """ + + grid_mapping_variables_coordinates = set() + for ncvar in ds.get_variables_by_attributes(grid_mapping=lambda x: x is not None and isinstance(x, str)): + + if ' ' in ncvar.grid_mapping: + coord_mapping_list = [ + x.split(':')[0] for x in ncvar.grid_mapping.split(' ') if not ':' in x + ] + if coord_mapping_list: + for coord_item in coord_mapping_list: + if coord_item in ds.variables: + grid_mapping_variables_coordinates.add(coord_item) + + return grid_mapping_variables_coordinates @lru_cache(128) def get_axis_map(ds, variable): diff --git a/compliance_checker/tests/test_cf.py b/compliance_checker/tests/test_cf.py index e3b569563..7c9dcc39b 100644 --- a/compliance_checker/tests/test_cf.py +++ b/compliance_checker/tests/test_cf.py @@ -1293,16 +1293,18 @@ def test_check_reduced_horizontal_grid(self): assert len([r for r in results if r.value[0] < r.value[1]]) == 1 assert all(r.name == "§5.3 Reduced Horizontal Grid" for r in results) -# def test_check_grid_mapping(self): -# dataset = self.load_dataset(STATIC_FILES["mapping"]) -# results = self.cf.check_grid_mapping(dataset) - -# assert len(results) == 6 -# assert len([r.value for r in results.values() if r.value[0] < r.value[1]]) == 0 -# expected_name = ( -# "§5.6 Horizontal Coordinate Reference Systems, Grid Mappings, Projections" -# ) -# assert all(r.name == expected_name for r in results.values()) + # def test_check_grid_mapping(self): + # dataset = self.load_dataset(STATIC_FILES["mapping"]) + # # print(dataset.variables) + # results = self.cf.check_grid_mapping(dataset) + # print(results.values) + + # assert len(results) == 6 + # assert len([r.value for r in results.values() if r.value[0] < r.value[1]]) == 0 + # expected_name = ( + # "§5.6 Horizontal Coordinate Reference Systems, Grid Mappings, Projections" + # ) + # assert all(r.name == expected_name for r in results.values()) def test_is_geophysical(self): @@ -1946,50 +1948,80 @@ def test_cell_measures(self): assert message in messages def test_check_grid_mapping(self): - # dataset = self.load_dataset(STATIC_FILES["mapping"]) - # results = self.cf.check_grid_mapping(dataset) - - # assert len(results) == 6 - # assert len([r.value for r in results.values() if r.value[0] < r.value[1]]) == 0 - # expected_name = ( - # "§5.6 Horizontal Coordinate Reference Systems, Grid Mappings, Projections" - # ) - # assert all(r.name == expected_name for r in results.values()) - - # create Cells with grid_apping variable - dataset = MockTimeSeries() + # create cells with grid_apping variable + dataset = MockTimeSeries() - dataset.createVariable("temp", "d", ("time")) - dataset.createVariable("crsOSGB", "d") - dataset.createVariable("crsWGS84", "d") + # add variable to the file as auxilary coordinate + # NOTE: Dimension variable It is a one-dimensional variable with the same name as its dimension [e.g., + # time(time) ], and it is defined as a numeric data type with values that are + # ordered monotonically. Missing values are not allowed in coordinate variables. + # + dataset.createDimension("y", 500) + dataset.createDimension("x", 500) + dataset.createVariable("y", "d", ("y",)) + dataset.createVariable("x", "d", ("x",)) + y = dataset.variables["y"] + y.axis = "Y" + y.standard_name = "projection_y_coordinate" + x = dataset.variables["x"] + x.axis = "X" + x.standard_name = "projection_x_coordinate" + + # add a variable with "grid_mapping" as an attribute + dataset.createVariable("temp", "d", ("time", "x", "y")) + dataset.createVariable("crsOSGB", "i") + dataset.createVariable("crsWGS84", "i") temp = dataset.variables["temp"] temp.standard_name = "air_temperature" temp.units = "K" temp.coordinates = "lat lon" - temp.grid_mapping = "crsOSGB: time crsWGS84: lat lon" + temp.grid_mapping = "crsOSGB: z y crsWGS84: lat lon" # create grid_mapping crsOSGB ; crsOSGB = dataset.variables["crsOSGB"] crsOSGB.grid_mapping_name = "transverse_mercator" - crsOSGB.semi_major_axis = 6377563.396 + crsOSGB.semi_major_axis = "6377563.396" crsOSGB.inverse_flattening = 299.3249646 - crsOSGB.ongitude_of_prime_meridian = 0.0 + crsOSGB.longitude_of_prime_meridian = 0.0 crsOSGB.latitude_of_projection_origin = 49.0 crsOSGB.longitude_of_central_meridian = -2.0 - crsOSGB.scale_factor_at_central_meridian = 0.9996012717 + crsOSGB.scale_factor_at_central_meridian = "0.9996012717" crsOSGB.false_easting = 400000.0 crsOSGB.false_northing = -100000.0 crsOSGB.unit = "metre" - + # create grid_mapping crsWGS84 crsWGS84 = dataset.variables["crsWGS84"] crsWGS84.grid_mapping_name = "latitude_longitude" crsWGS84.longitude_of_prime_meridian = 0.0 crsWGS84.semi_major_axis = 6378137.0 crsWGS84.inverse_flattening = 298.257223563 + crsWGS84.reference_ellipsoid_name = "testing" + crsWGS84.prime_meridian_name = "testing" + crsWGS84.horizontal_datum_name = "testing" + crsWGS84.geographic_crs_name = "testing" + crsWGS84.projected_crs_name = "testing" results = self.cf.check_grid_mapping(dataset) + assert len(results) == 3 + assert len([r.value for r in results.values() if r.value[0] < r.value[1]]) == 1 + expected_name = ( + "§5.6 Horizontal Coordinate Reference Systems, Grid Mappings, Projections" + ) + assert all(r.name == expected_name for r in results.values()) + + # test a different datastet + dataset = self.load_dataset(STATIC_FILES["mapping"]) + # print(dataset.variables) + results = self.cf.check_grid_mapping(dataset) + assert len(results) == 6 + assert len([r.value for r in results.values() if r.value[0] < r.value[1]]) == 1 + expected_name = ( + "§5.6 Horizontal Coordinate Reference Systems, Grid Mappings, Projections" + ) + assert all(r.name == expected_name for r in results.values()) + def test_variable_features(self): with MockTimeSeries() as dataset: From be02e316d25b66c32258c4d210a77fbd616639a4 Mon Sep 17 00:00:00 2001 From: Leila Belabbassi Date: Tue, 24 Jan 2023 10:23:21 -0600 Subject: [PATCH 06/12] Update test_cf.py removed commented function --- compliance_checker/tests/test_cf.py | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/compliance_checker/tests/test_cf.py b/compliance_checker/tests/test_cf.py index 7c9dcc39b..b63320bff 100644 --- a/compliance_checker/tests/test_cf.py +++ b/compliance_checker/tests/test_cf.py @@ -1293,19 +1293,6 @@ def test_check_reduced_horizontal_grid(self): assert len([r for r in results if r.value[0] < r.value[1]]) == 1 assert all(r.name == "§5.3 Reduced Horizontal Grid" for r in results) - # def test_check_grid_mapping(self): - # dataset = self.load_dataset(STATIC_FILES["mapping"]) - # # print(dataset.variables) - # results = self.cf.check_grid_mapping(dataset) - # print(results.values) - - # assert len(results) == 6 - # assert len([r.value for r in results.values() if r.value[0] < r.value[1]]) == 0 - # expected_name = ( - # "§5.6 Horizontal Coordinate Reference Systems, Grid Mappings, Projections" - # ) - # assert all(r.name == expected_name for r in results.values()) - def test_is_geophysical(self): # check whether string type variable, which are not `cf_role`, are @@ -1472,15 +1459,6 @@ def test_compress_packed(self): self.assertFalse(results[0].value) self.assertFalse(results[1].value) - # def test_check_all_features_are_same_type(self): - # dataset = self.load_dataset(STATIC_FILES["rutgers"]) - # result = self.cf.check_all_features_are_same_type(dataset) - # assert result - - # dataset = self.load_dataset(STATIC_FILES["featureType"]) - # result = self.cf.check_all_features_are_same_type(dataset) - # assert result - def test_featureType_is_case_insensitive(self): """ Tests that the featureType attribute is case insensitive From 9286cf5dbca16ab69c171ed6a1edef9fc027678c Mon Sep 17 00:00:00 2001 From: Leila Belabbassi Date: Tue, 24 Jan 2023 12:09:59 -0600 Subject: [PATCH 07/12] Update cf_1_7.py removed "print( )" to suppress output, it is needed only for debugging. removed the IF conditional statement and used the tests' results from _check_gmattr_existence_condition_ell_pmerid_hdatum and _check_gmattr_existence_condition_geoid_name_geoptl_datum_name --- compliance_checker/cf/cf_1_7.py | 35 +++++++-------------------------- 1 file changed, 7 insertions(+), 28 deletions(-) diff --git a/compliance_checker/cf/cf_1_7.py b/compliance_checker/cf/cf_1_7.py index 85ea6a5d9..95da2afa0 100644 --- a/compliance_checker/cf/cf_1_7.py +++ b/compliance_checker/cf/cf_1_7.py @@ -242,7 +242,6 @@ def check_cell_boundaries(self, ds): ds ).items(): - print(boundary_variable_name, boundary_variable_name.dtype) variable = ds.variables[variable_name] valid = True reasoning = [] @@ -847,46 +846,26 @@ def check_grid_mapping(self, ds): test_ctx.out_of += 1 test_ctx.score += 1 - # existence_conditions + # existence_conditions + # Both geoid_name and geopotential_datum_name cannot exist exist_cond_1 = ( self._check_gmattr_existence_condition_geoid_name_geoptl_datum_name(var) ) - if exist_cond_1[0] == False: - test_ctx.messages.append("Both geoid_name and geopotential_datum_name " - "should not exist for {}".format(var.name)) - test_ctx.out_of += 1 - + test_ctx.messages.append(exist_cond_1) test_ctx.score += 1 - # test_ctx.assert_true(exist_cond_1[0], exist_cond_1[1]) # [8/9] reference_ellipsoid_name, prime_meridian_name, horizontal_datum_name and # geographic_crs_name must be all defined if any one is defined. exist_cond_2 = self._check_gmattr_existence_condition_ell_pmerid_hdatum(var) - if exist_cond_2[0] == False: - test_ctx.messages.append("reference_ellipsoid_name, prime_meridian_name, " - "horizontal_datum_name and geographic_crs_name must be all defined " - "if any one is defined") - test_ctx.out_of += 1 - - test_ctx.score += 1 - # test_ctx.assert_true(exist_cond_2[0], exist_cond_2[1]) + test_ctx.messages.append(exist_cond_2) + test_ctx.score += 1 # handle vertical datum related grid_mapping attributes vert_datum_attrs = {} possible_vert_datum_attrs = {"geoid_name", "geopotential_datum_name"} - vert_datum_attrs = possible_vert_datum_attrs.intersection(var.ncattrs()) - len_vdatum_name_attrs = len(vert_datum_attrs) + vert_datum_attrs = possible_vert_datum_attrs.intersection(var.ncattrs()) - # check that geoid_name and geopotential_datum_name are not both - # present in the grid_mapping variable - if len_vdatum_name_attrs == 2: - test_ctx.out_of += 1 - test_ctx.messages.append( - "Cannot have both 'geoid_name' and " - "'geopotential_datum_name' attributes in " - "grid mapping variable '{}'".format(var.name) - ) - elif len_vdatum_name_attrs == 1: + if exist_cond_1[0] == True and vert_datum_attrs: # should be one or zero attrs proj_db_path = os.path.join(pyproj.datadir.get_data_dir(), "proj.db") try: From 0f0fa24945751d712d242ac786ae62a7a48508ff Mon Sep 17 00:00:00 2001 From: Leila Belabbassi Date: Mon, 1 May 2023 16:40:57 -0500 Subject: [PATCH 08/12] Update cf_1_7.py using regex I have replaced an inner loop with: grid_mapping_variable_list.append(regex.findall(r"(\w+)",item.grid_mapping)) --- compliance_checker/cf/cf_1_7.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/compliance_checker/cf/cf_1_7.py b/compliance_checker/cf/cf_1_7.py index 95da2afa0..f41669fcd 100644 --- a/compliance_checker/cf/cf_1_7.py +++ b/compliance_checker/cf/cf_1_7.py @@ -741,13 +741,11 @@ def check_grid_mapping(self, ds): test_ctx.messages.append("The grid_mapping attribute {} " "is not a string".format(item.grid_mapping)) test_ctx.out_of += 1 - if ' ' in item.grid_mapping: - for x in item.grid_mapping.replace(":", '').split(): - grid_mapping_variable_list.append(x) - else: - # [2/9] Note that in its simplest form the attribute comprises just - # a grid_mapping_name as a single word. - grid_mapping_variable_list.append(item.grid_mapping) + + # [2/9] Note that in its simplest form the attribute comprises just + # a grid_mapping_name as a single word. + grid_mapping_variable_list.append(regex.findall(r"(\w+)",item.grid_mapping)) + # # Get the CF grid_mapping variables list (== only if the variable exist in the Variables list) grid_mapping_variables = cfutil.get_grid_mapping_variables(ds) From 177513ac1156dfe3559b71f93523ddc5417758ac Mon Sep 17 00:00:00 2001 From: Leila Belabbassi Date: Sun, 14 May 2023 09:28:50 -0500 Subject: [PATCH 09/12] Update cf_1_7.py added the "else" clause to the "if not" statement on line 740 --- compliance_checker/cf/cf_1_7.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compliance_checker/cf/cf_1_7.py b/compliance_checker/cf/cf_1_7.py index f41669fcd..62b17bfba 100644 --- a/compliance_checker/cf/cf_1_7.py +++ b/compliance_checker/cf/cf_1_7.py @@ -741,7 +741,7 @@ def check_grid_mapping(self, ds): test_ctx.messages.append("The grid_mapping attribute {} " "is not a string".format(item.grid_mapping)) test_ctx.out_of += 1 - + else: # [2/9] Note that in its simplest form the attribute comprises just # a grid_mapping_name as a single word. grid_mapping_variable_list.append(regex.findall(r"(\w+)",item.grid_mapping)) From 84a08ecaf00776d568db301e703cf7485e911700 Mon Sep 17 00:00:00 2001 From: Benjamin Adams Date: Tue, 21 May 2024 16:45:21 -0400 Subject: [PATCH 10/12] Fixup unused parameter name in cfutil get_grid_mapping_variables --- compliance_checker/cfutil.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compliance_checker/cfutil.py b/compliance_checker/cfutil.py index e5042b1d0..53fc93570 100644 --- a/compliance_checker/cfutil.py +++ b/compliance_checker/cfutil.py @@ -766,7 +766,7 @@ def get_flag_variables(nc): return flag_variables -def get_grid_mapping_variables(nc): +def get_grid_mapping_variables(ds): """ Returns a list of grid mapping variables From e5ba8be27d83c1b03c8461d97e44ffb6360509d4 Mon Sep 17 00:00:00 2001 From: Benjamin Adams Date: Tue, 21 May 2024 16:50:51 -0400 Subject: [PATCH 11/12] FIXUP: More undef fixes --- compliance_checker/cfutil.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compliance_checker/cfutil.py b/compliance_checker/cfutil.py index 53fc93570..48d9ebdc5 100644 --- a/compliance_checker/cfutil.py +++ b/compliance_checker/cfutil.py @@ -787,8 +787,8 @@ def get_grid_mapping_variables(ds): if ncvar.grid_mapping in ds.variables: grid_mapping_variables.add(ncvar.grid_mapping) - for ncvar in nc.get_variables_by_attributes(grid_mapping=lambda x: x is not None): - if ncvar.grid_mapping in nc.variables: + for ncvar in ds.get_variables_by_attributes(grid_mapping=lambda x: x is not None): + if ncvar.grid_mapping in ds.variables: grid_mapping_variables.add(ncvar.grid_mapping) return grid_mapping_variables From 579a0183cdeb86ba189111ba9e2ecaa1e1e24f2b Mon Sep 17 00:00:00 2001 From: Benjamin Adams Date: Tue, 21 May 2024 17:01:26 -0400 Subject: [PATCH 12/12] FIXUP: Add missing regex module --- compliance_checker/cf/cf_1_7.py | 1 + 1 file changed, 1 insertion(+) diff --git a/compliance_checker/cf/cf_1_7.py b/compliance_checker/cf/cf_1_7.py index 46a9fc9a4..8a9756425 100644 --- a/compliance_checker/cf/cf_1_7.py +++ b/compliance_checker/cf/cf_1_7.py @@ -6,6 +6,7 @@ import numpy as np import pyproj +import regex from compliance_checker import cfutil from compliance_checker.base import BaseCheck, Result, TestCtx from compliance_checker.cf.appendix_d import dimless_vertical_coordinates_1_7