diff --git a/pyaerocom/aeroval/bulkfraction_engine.py b/pyaerocom/aeroval/bulkfraction_engine.py new file mode 100644 index 000000000..ceb49e2a7 --- /dev/null +++ b/pyaerocom/aeroval/bulkfraction_engine.py @@ -0,0 +1,214 @@ +import logging + +from pyaerocom import ColocatedData +from pyaerocom.aeroval._processing_base import HasColocator, ProcessingEngine +from pyaerocom.aeroval.obsentry import ObsEntry +from pyaerocom.aeroval.modelentry import ModelEntry +from pyaerocom.aeroval.coldatatojson_engine import ColdataToJsonEngine + + +from pyaerocom.colocation.colocator import Colocator +from pyaerocom.colocation.colocation_setup import ColocationSetup + + +logger = logging.getLogger(__name__) + + +class BulkFractionEngine(ProcessingEngine, HasColocator): + def run(self, var_list: list[str] | str | None, model_name: str, obs_name: str): + self.sobs_cfg = self.cfg.obs_cfg.get_entry(obs_name) + self.smodel_cfg = self.cfg.model_cfg.get_entry(model_name) + + if var_list is None: + var_list = self.sobs_cfg.obs_vars + elif isinstance(var_list, str): + var_list = [var_list] + elif not isinstance(var_list, list): + raise ValueError(f"invalid input for var_list: {var_list}.") + + files_to_convert = [] + for var_name in var_list: + bulk_vars = self._get_bulk_vars(var_name, self.sobs_cfg) + + freq = self.sobs_cfg.ts_type + cd, fp = self._run_var( + model_name, obs_name, var_name, bulk_vars, freq, self.sobs_cfg, self.smodel_cfg + ) + + files_to_convert.append(fp) + + engine = ColdataToJsonEngine(self.cfg) + engine.run(files_to_convert) + + def _get_bulk_vars(self, var_name: str, cfg: ObsEntry) -> list: + bulk_vars = cfg.bulk_options + if var_name not in bulk_vars: + raise KeyError(f"Could not find bulk vars entry for {var_name}") + + if len(bulk_vars[var_name].vars) != 2: + raise ValueError( + f"(Only) 2 entries must be present for bulk vars to calculate fraction for {var_name}. Found {bulk_vars[var_name]}" + ) + + return bulk_vars[var_name].vars + + def _run_var( + self, + model_name: str, + obs_name: str, + var_name: str, + bulk_vars: list, + freq: str, + obs_entry: ObsEntry, + model_entry: ModelEntry, + ) -> tuple[ColocatedData, str]: + model_exists = obs_entry.bulk_options[var_name].model_exists + + cols = self.get_colocators(bulk_vars, var_name, freq, model_name, obs_name, model_exists) + + coldatas = [] + for col in cols: + if len(list(col.keys())) != 1: + raise ValueError( + "Found more than one colocated object when trying to run bulk variable" + ) + bv = list(col.keys())[0] + coldatas.append(col[bv].run(bv)) + + num_name, denum_name = bulk_vars[0], bulk_vars[1] + num_col = cols[0][num_name].run(num_name) + denum_col = cols[1][denum_name].run(denum_name) + + model_num_name, model_denum_name = self._get_model_var_names( + var_name, bulk_vars, model_exists, model_entry + ) + + cd = self._combine_coldatas( + num_col[model_num_name][num_name], + denum_col[model_denum_name][denum_name], + var_name, + obs_entry, + ) + + num_colocator = cols[0][num_name] + + fp = cd.to_netcdf( + out_dir=num_colocator.output_dir, + savename=cd._aerocom_savename( + var_name, + obs_name, + var_name, + model_name, + num_colocator.get_start_str(), + num_colocator.get_stop_str(), + freq, + num_colocator.colocation_setup.filter_name, + None, # cd.data.attrs["vert_code"], + ), + ) + return cd, fp + + def _combine_coldatas( + self, + num_coldata: ColocatedData, + denum_coldata: ColocatedData, + var_name: str, + obs_entry: ObsEntry, + ) -> ColocatedData: + mode = obs_entry.bulk_options[var_name].mode + model_exists = obs_entry.bulk_options[var_name].model_exists + units = obs_entry.bulk_options[var_name].units + + if mode == "fraction": + new_data = num_coldata.data / denum_coldata.data + + elif mode == "product": + new_data = num_coldata.data * denum_coldata.data + else: + raise ValueError(f"Mode must be either fraction of product, and not {mode}") + if model_exists: + # TODO: Unsure if this works!!! + new_data[1] = num_coldata.data[1].where(new_data[1]) + + cd = ColocatedData(new_data) + + cd.data.attrs = num_coldata.data.attrs + cd.data.attrs["var_name"] = [var_name, var_name] + cd.data.attrs["var_units"] = [units, units] + cd.metadata["var_name_input"] = [var_name, var_name] + return cd + + def _get_model_var_names( + self, var_name: str, bulk_vars: list[str], model_exists: bool, model_entry: ModelEntry + ) -> tuple[str]: + num_name, denum_name = bulk_vars[0], bulk_vars[1] + if model_exists: + num_name, denum_name = var_name, var_name + + model_use_vars = model_entry.model_use_vars + if model_use_vars != {}: + num_name, denum_name = model_use_vars[num_name], model_use_vars[denum_name] + + return num_name, denum_name + + def get_colocators( + self, + bulk_vars: list, + var_name: str, + freq: str, + model_name: str = None, + obs_name: str = None, + model_exists: bool = False, + ) -> list[dict[str | Colocator]]: + """ + Instantiate colocation engine + + Parameters + ---------- + model_name : str, optional + name of model. The default is None. + obs_name : str, optional + name of obs. The default is None. + + Returns + ------- + Colocator + + """ + col_cfg = {**self.cfg.colocation_opts.model_dump()} + outdir = self.cfg.path_manager.get_coldata_dir() + col_cfg["basedir_coldata"] = outdir + + if model_name: + mod_cfg = self.cfg.get_model_entry(model_name) + col_cfg["model_cfg"] = mod_cfg + + # Hack and at what lowlevel_helpers's import_from was doing + for key, val in mod_cfg.items(): + if key in ColocationSetup.model_fields: + col_cfg[key] = val + if obs_name: + obs_cfg = self.cfg.get_obs_entry(obs_name) + pyaro_config = obs_cfg["obs_config"] if "obs_config" in obs_cfg else None + col_cfg["obs_config"] = pyaro_config + + # Hack and at what lowlevel_helpers's import_from was doing + for key, val in obs_cfg.model_dump().items(): + if key in ColocationSetup.model_fields: + col_cfg[key] = val + + col_cfg["add_meta"].update(diurnal_only=self.cfg.get_obs_entry(obs_name).diurnal_only) + cols = [] + col_cfg["ts_type"] = freq + for bulk_var in bulk_vars: + col_cfg["obs_vars"] = bulk_var + if model_exists: + col_cfg["model_use_vars"] = { + bulk_var: var_name, + } + col_stp = ColocationSetup(**col_cfg) + col = Colocator(col_stp) + + cols.append({bulk_var: col}) + + return cols diff --git a/pyaerocom/aeroval/data/var_scale_colmap.ini b/pyaerocom/aeroval/data/var_scale_colmap.ini index 7d61272f8..f0c582e77 100644 --- a/pyaerocom/aeroval/data/var_scale_colmap.ini +++ b/pyaerocom/aeroval/data/var_scale_colmap.ini @@ -440,4 +440,29 @@ colmap = coolwarm [vcdno2] scale = [0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120] +colmap = coolwarm + + +[fractioneBCbb] +scale = [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0] +colmap = coolwarm + +[fractioneBCff] +scale = [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0] +colmap = coolwarm + +[concebc] +scale = [0.0, 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.5, 1.0, 2.0] +colmap = coolwarm + +[concebcem] +scale = [0.0, 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.5, 1.0, 2.0] +colmap = coolwarm + +[fractioneBCbbem] +scale = [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0] +colmap = coolwarm + +[fractioneBCffem] +scale = [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0] colmap = coolwarm \ No newline at end of file diff --git a/pyaerocom/aeroval/data/var_web_info.ini b/pyaerocom/aeroval/data/var_web_info.ini index fbdf17585..1461a188e 100644 --- a/pyaerocom/aeroval/data/var_web_info.ini +++ b/pyaerocom/aeroval/data/var_web_info.ini @@ -847,3 +847,52 @@ category = Vertical column density menu_name = VCD CO vertical_type = 3D category = Vertical column density + + +; # Fraction tests +; [pmfraction] +; menu_name = PM Fraction +; vertical_type = 3D +; category = Particle ratio + +; [pmfraction2] +; menu_name = PM Fraction 2 +; vertical_type = 3D +; category = Particle ratio + +[concwetrdn] +menu_name = RDN in Precip +vertical_type = 3D +category = Particle concentrations + + +# EC +; [fractionECFine] +; menu_name = EC Fine Fraction +; vertical_type = 3D +; category = Particle ratio + +; [fractionECCoarse] +; menu_name = EC Fine Fraction +; vertical_type = 3D +; category = Particle ratio + +; [fractionEC] +; menu_name = EC Fraction +; vertical_type = 3D +; category = Particle ratio + +[fractioneBCbb] +menu_name = eBC Residential Fraction +vertical_type = 3D +category = Particle ratio + +[fractioneBCff] +menu_name = eBC Non-Residential Fraction +vertical_type = 3D +category = Particle ratio + +[concebc] +menu_name = Equivalent BC +vertical_type = 3D +category = Particle concentrations diff --git a/pyaerocom/aeroval/experiment_processor.py b/pyaerocom/aeroval/experiment_processor.py index 5275eba93..74580c15e 100644 --- a/pyaerocom/aeroval/experiment_processor.py +++ b/pyaerocom/aeroval/experiment_processor.py @@ -8,6 +8,7 @@ from pyaerocom.aeroval.helpers import delete_dummy_model, make_dummy_model from pyaerocom.aeroval.modelmaps_engine import ModelMapsEngine from pyaerocom.aeroval.superobs_engine import SuperObsEngine +from pyaerocom.aeroval.bulkfraction_engine import BulkFractionEngine logger = logging.getLogger(__name__) @@ -65,6 +66,10 @@ def _run_single_entry(self, model_name, obs_name, var_list): engine = ColdataToJsonEngine(self.cfg) engine.run(files_to_convert) + elif ocfg.is_bulk: + engine = BulkFractionEngine(self.cfg) + engine.run(var_list, model_name, obs_name) + else: # If a var_list is given, only run on the obs networks which contain that variable if var_list: diff --git a/pyaerocom/aeroval/obsentry.py b/pyaerocom/aeroval/obsentry.py index 6b1e8302c..498637a00 100644 --- a/pyaerocom/aeroval/obsentry.py +++ b/pyaerocom/aeroval/obsentry.py @@ -37,6 +37,22 @@ ) +class BulkOptions(BaseModel): + #: Vars to be used to calculate fraction or product + vars: tuple[str, str] + #: Whether or not the bulk variable exist as a model var. If not, it will be calculated same way as obs vars + model_exists: bool + #: Is the result a product or fraction + mode: Literal["product", "fraction"] + #: Unit of result + units: str + + model_config = ConfigDict( + validate_assignment=True, + protected_namespaces=(), + ) + + class ObsEntry(BaseModel): """Observation configuration for evaluation (BaseModel) @@ -75,6 +91,9 @@ class ObsEntry(BaseModel): only_superobs : bool this indicates whether this configuration is only to be used as part of a superobs network, and not individually. + is_bulkfraction: bool + If true numerator and denominator are colocated separately, before the fraction is calculated. + For this to work, the numerator and denominator need to be given in bulk_options read_opts_ungridded : :obj:`dict`, optional dictionary that specifies reading constraints for ungridded reading (c.g. :class:`pyaerocom.io.ReadUngridded`). @@ -96,6 +115,8 @@ class ObsEntry(BaseModel): validate_assignment=True, ) + ## Pydantic structs + ###################### ## Required attributes ###################### @@ -111,6 +132,8 @@ class ObsEntry(BaseModel): instr_vert_loc: str | None = None is_superobs: bool = False only_superobs: bool = False + is_bulk: bool = False + bulk_options: dict[str, BulkOptions] = {} colocation_layer_limts: tuple[LayerLimits, ...] | None = None profile_layer_limits: tuple[LayerLimits, ...] | None = None web_interface_name: str | None = None @@ -169,9 +192,7 @@ def check_obs_vert_type(cls, ovt): ovt = ALT_NAMES_VERT_CODES[ovt] return ovt valid = SUPPORTED_VERT_CODES + list(ALT_NAMES_VERT_CODES) - raise ValueError( - f"Invalid value for obs_vert_type: {ovt}. " f"Supported codes are {valid}." - ) + raise ValueError(f"Invalid value for obs_vert_type: {ovt}. Supported codes are {valid}.") @model_validator(mode="after") def check_cfg(self): @@ -180,6 +201,16 @@ def check_cfg(self): f"Invalid value for obs_id: {self.obs_id}. Need str, tuple, or dict " f"or specification of ids and variables via obs_compute_post" ) + if self.is_bulk: + for var in self.obs_vars: + if var not in self.bulk_options: + raise KeyError(f"Could not find bulk vars entry for {var}") + + elif len(self.bulk_options[var].vars) != 2: + raise ValueError( + f"(Only) 2 entries must be present for bulk vars to calculate fraction for {var}" + ) + self.check_add_obs() return self @@ -236,6 +267,6 @@ def get_vert_code(self, var): raise ValueError(f"invalid value for obs_vert_type: {vc}") if val not in SUPPORTED_VERT_CODES: raise ValueError( - f"invalid value for obs_vert_type: {val}. Choose from " f"{SUPPORTED_VERT_CODES}." + f"invalid value for obs_vert_type: {val}. Choose from {SUPPORTED_VERT_CODES}." ) return val diff --git a/pyaerocom/aux_var_helpers.py b/pyaerocom/aux_var_helpers.py index 31f3deb3e..10da4ea49 100644 --- a/pyaerocom/aux_var_helpers.py +++ b/pyaerocom/aux_var_helpers.py @@ -294,7 +294,10 @@ def _calc_od_helper( ods_alt = data[od_ref_alt][mask] ang = data[use_angstrom_coeff][mask] replace = compute_od_from_angstromexp( - to_lambda=to_lambda, od_ref=ods_alt, lambda_ref=lambda_ref_alt, angstrom_coeff=ang + to_lambda=to_lambda, + od_ref=ods_alt, + lambda_ref=lambda_ref_alt, + angstrom_coeff=ang, ) result[mask] = replace if treshold_angstrom: @@ -540,6 +543,53 @@ def _compute_wdep_from_concprcp_helper(data, wdep_var, concprcp_var, pr_var): return wdep +def _compute_wdeppr_from_concprcp_helper(data, wdep_pr_var): + pr_var = "pr" + vars_needed = [pr_var] + + if not all(x in data.data_flagged for x in vars_needed): + raise ValueError(f"Need flags for {vars_needed} to compute wet deposition") + from pyaerocom import TsType + from pyaerocom.units_helpers import RATES_FREQ_DEFAULT, get_unit_conversion_fac + + tst = TsType(data.get_var_ts_type(pr_var)) + + ival = tst.to_si() + + pr_unit = data.get_unit(pr_var) + if not pr_unit == "m": + data.convert_unit(pr_var, "m") + pr_unit = data.get_unit(pr_var) + pr_data = data[pr_var] + pr_flags = data.data_flagged[pr_var] + # check where precip data is zero (it did not rain!) and for each of + # these timestamps, set concprcp to 0 (it should be 0 if there is no + # rain...) and set flags in concprcp to False (these data are to be used + # later) + pr_zero = pr_data == 0 + if pr_zero.sum() > 0: + pr_flags[pr_zero] = False + wdep_pr = pr_data + + if not ival == RATES_FREQ_DEFAULT: + fac = get_unit_conversion_fac(ival, RATES_FREQ_DEFAULT) + wdep_pr /= fac + # in units of ts_type, that is, e.g. kg m-2 d + freq_str = f" {RATES_FREQ_DEFAULT}-1" + pr_unit += freq_str + + if wdep_pr_var not in data.var_info: + data.var_info[wdep_pr_var] = {} + data.var_info[wdep_pr_var]["units"] = pr_unit + + # set flags for wetso4 + wdep_flags = np.zeros(len(wdep_pr)).astype(bool) + wdep_flags[pr_flags] = True + data.data_flagged[wdep_pr_var] = wdep_flags + + return wdep_pr + + def compute_wetoxs_from_concprcpoxs(data): """Compute wdep from conc in precip and precip data @@ -563,6 +613,10 @@ def compute_wetoxs_from_concprcpoxs(data): return _compute_wdep_from_concprcp_helper(data, "wetoxs", "concprcpoxs", "pr") +def compute_wetoxspr_from_concprcpoxs(data): + return _compute_wdeppr_from_concprcp_helper(data, "wetoxspr") + + def compute_wetoxs_from_concprcpoxst(data): """Compute wdep from conc in precip and precip data @@ -655,6 +709,10 @@ def compute_wetrdn_from_concprcprdn(data): return _compute_wdep_from_concprcp_helper(data, "wetrdn", "concprcprdn", "pr") +def compute_wetrdnpr_from_concprcprdn(data): + return _compute_wdeppr_from_concprcp_helper(data, "wetrdnpr") + + def compute_wetnh4_from_concprcpnh4(data): return _compute_wdep_from_concprcp_helper(data, "wetnh4", "concprcpnh4", "pr") @@ -671,6 +729,10 @@ def compute_wetna_from_concprcpna(data): return _compute_wdep_from_concprcp_helper(data, "wetna", "concprcpna", "pr") +def compute_wetso4pr_from_concprcpso4(data): + return _compute_wdeppr_from_concprcp_helper(data, "wetso4pr") + + def vmrx_to_concx(data, p_pascal, T_kelvin, vmr_unit, mmol_var, mmol_air=None, to_unit=None): """ Convert volume mixing ratio (vmr) to mass concentration diff --git a/pyaerocom/colocation/colocator.py b/pyaerocom/colocation/colocator.py index 31bd78315..ff48ee4d6 100644 --- a/pyaerocom/colocation/colocator.py +++ b/pyaerocom/colocation/colocator.py @@ -950,6 +950,9 @@ def _infer_start_stop_yr_from_model_reader(self): def _check_set_start_stop(self): if self.colocation_setup.start is None: self._infer_start_stop_yr_from_model_reader() + start, stop = self.start, self.stop + else: + start, stop = self.colocation_setup.start, self.colocation_setup.stop if self.colocation_setup.model_use_climatology: if self.colocation_setup.stop is not None or not isinstance( self.colocation_setup.start, int @@ -959,7 +962,7 @@ def _check_set_start_stop(self): 'climatology fields, please specify "start" as integer ' 'denoting the year, and set "stop"=None' ) - self.start, self.stop = start_stop(self.colocation_setup.start, self.colocation_setup.stop) + self.start, self.stop = start_stop(start, stop) def _coldata_savename(self, obs_var, mod_var, ts_type, **kwargs): """Get filename of colocated data file for saving""" diff --git a/pyaerocom/data/ebas_config.ini b/pyaerocom/data/ebas_config.ini index 8dbf69c8f..1e1377c9d 100644 --- a/pyaerocom/data/ebas_config.ini +++ b/pyaerocom/data/ebas_config.ini @@ -185,6 +185,11 @@ component=equivalent_black_carbon instrument=filter_absorption_photometer matrix=aerosol,pm1,pm10,pm25 +[concebc] +component=equivalent_black_carbon +instrument=filter_absorption_photometer +matrix=aerosol,pm1,pm10,pm25 + [concCec] component=elemental_carbon # after discussion with Wenche @@ -437,6 +442,15 @@ matrix=precip component=precipitation_amount_off,precipitation_amount matrix=precip +# 4.1 Precipitation for given wetdep +[wetoxspr] +requires=concprcpoxs + +[wetso4pr] +requires=concprcpso4 + + + [concCocpm10] component=organic_carbon matrix=pm10 @@ -654,3 +668,6 @@ requires=concprcpoxs ; [conclevoglucosan] ; component=levoglucosan, C6H10O5, LVG, levo_ng/m3, Levoglucosan ; matrix=pm10,aerosol,pm25 + +[wetrdnpr] +requires=concprcprdn \ No newline at end of file diff --git a/pyaerocom/data/variables.ini b/pyaerocom/data/variables.ini index e9236d1d4..d11965396 100644 --- a/pyaerocom/data/variables.ini +++ b/pyaerocom/data/variables.ini @@ -2765,7 +2765,7 @@ var_type = mass concentration [concprcprdn] description = Mass concentration of reduced nitrogen in precipitation -unit = ug N m-3 +unit = mg N m-3 var_type = mass concentration [concso4] @@ -6033,3 +6033,132 @@ unit = mol m-2 minimum = 0 maximum = 100000000 dimensions = time,lat,lon + + +# Preciptitation for given wetdep +[wetrdnpr] +description=Precipitation measured with reduced Nitrogen mass +unit = m d-1 +minimum=0 + +[wetoxspr] +description=Precipitation measured with total sulfur mass +unit = m d-1 +minimum=0 + +[wetso4pr] +description=Precipitation measured with SO4 +unit = m d-1 +minimum=0 + +[concwetrdn] +description=Concentration of reduced Nitrogen mass in precipitation +unit = mg N m-3 + + +# EC variables + +[concecFineRes] +description=Mass concentration of fine elemental carbon residental +unit = ug m-3 + +[concecFineNonRes] +description=Mass concentration of fine elemental carbon non-residental +unit = ug m-3 + +[concecCoarseRes] +description=Mass concentration of coars elemental carbon residental +unit = ug m-3 + +[concecCoarseNonRes] +description=Mass concentration of coarse elemental carbon non-residental +unit = ug m-3 + +[concecTotalRes] +description=Mass concentration of total elemental carbon residental +unit = ug m-3 + +[concecTotalNonRes] +description=Mass concentration of total elemental carbon non-residental +unit = ug m-3 + +# EC variables emission + +[concecFineResEM] +description=Mass concentration of fine elemental carbon residental, from emmissions +unit = ug m-3 + +[concecFineNonResEM] +description=Mass concentration of fine elemental carbon non-residental, from emmissions +unit = ug m-3 + +[concecCoarseResEM] +description=Mass concentration of coars elemental carbon residental, from emmissions +unit = ug m-3 + +[concecCoarseNonResEM] +description=Mass concentration of coarse elemental carbon non-residental, from emmissions +unit = ug m-3 + +[concecTotalResEM] +description=Mass concentration of total elemental carbon residental, from emmissions +unit = ug m-3 + +[concecTotalNonResEM] +description=Mass concentration of total elemental carbon non-residental, from emmissions +unit = ug m-3 + + +; [fractionECFine] +; description=Fraction of residental to non-residental fine elemental carbon +; unit = 1 + +; [fractionECCoarse] +; description=Fraction of residental to non-residental fine elemental carbon +; unit = 1 + +; [fractionEC] +; description=Fraction of residental to non-residental total elemental carbon +; unit = 1 +[fractioneBCbb] +description=Fraction of residental to total equivalent black carbon +unit = 1 +; maximum = 1 + +[fractioneBCff] +description=Fraction of non-residental to total equivalent black carbon +unit = 1 +; maximum = 1 + + +[concebc] +description=Mass concentration of equivalent black carbon +unit = ug m-3 +minimum = 0 +maximum = 200 + +[fractioneBCbbem] +description=Fraction of residental to total equivalent black carbon, from emmissions +unit = 1 +; maximum = 1 + +[fractioneBCffem] +description=Fraction of non-residental to total equivalent black carbon, from emmissions +unit = 1 +; maximum = 1 + + +[concebcem] +description=Mass concentration of equivalent black carbon, from emmissions +unit = ug m-3 +minimum = 0 +maximum = 200 + + +[concecFineEM] +description=Mass concentration of fine equivalent black carbon, from emmissions +unit = ug m-3 + +[concCecpm25EM] +description=Mass concentration of PM2.5 elemental carbon, from emmissions +unit = ug m-3 diff --git a/pyaerocom/io/mscw_ctm/emep_variables.toml b/pyaerocom/io/mscw_ctm/emep_variables.toml index 4b6d38182..a64ea1d8f 100644 --- a/pyaerocom/io/mscw_ctm/emep_variables.toml +++ b/pyaerocom/io/mscw_ctm/emep_variables.toml @@ -113,3 +113,27 @@ concoxn = "SURF_ugN_OXN" vmrno = "SURF_ppb_NO" vmrno2 = "SURF_ppb_NO2" concspores = "SURF_ug_FUNGAL_SPORES" + +# Test +pmfraction = "SURF_ug_PM10_rh50" +wetrdnpr = "WDEP_PREC" + +# Variables for EC +concecFineResNew = "SURF_ug_EC_F_RES_NEW" +concecFineResAge = "SURF_ug_EC_F_RES_AGE" +concecFineNonResNew = "SURF_ug_EC_F_NONRES_NEW" +concecFineNonResAge = "SURF_ug_EC_F_NONRES_AGE" +concecCoarseRes = "SURF_ug_EC_C_RES" +concecCoarseNonRes = "SURF_ug_EC_C_NONRES" + +# Variables for EC from emission +concecFineEM = "SURF_ug_EMECFINE" +concecCoarseEM = "SURF_ug_EMECCOARSE" + +concecFineResNewEM = "SURF_ug_EMEC_F_RES_NEW" +concecFineResAgeEM = "SURF_ug_EMEC_F_RES_AGE" +concecFineNonResNewEM = "SURF_ug_EMEC_F_NONRES_NEW" +concecFineNonResAgeEM = "SURF_ug_EMEC_F_NONRES_AGE" +concecCoarseResEM = "SURF_ug_EMEC_C_RES" +concecCoarseNonResEM = "SURF_ug_EMEC_C_NONRES" + diff --git a/pyaerocom/io/mscw_ctm/reader.py b/pyaerocom/io/mscw_ctm/reader.py index 1e66200b6..370af48be 100755 --- a/pyaerocom/io/mscw_ctm/reader.py +++ b/pyaerocom/io/mscw_ctm/reader.py @@ -117,6 +117,17 @@ class ReadMscwCtm(GriddedReader): "ratpm25pm10": ["concpm25", "concpm10"], # For Pollen # "concpolyol": ["concspores"], + # For EC + "concecFineRes": ["concecFineResNew", "concecFineResAge"], + "concecFineNonRes": ["concecFineNonResNew", "concecFineNonResAge"], + "concecTotalRes": ["concecFineRes", "concecCoarseRes"], + "concecTotalNonRes": ["concecFineNonRes", "concecCoarseNonRes"], + "concebc": ["concecFine", "concecCoarse"], + # For EC from emission + "concecTotalResEM": ["concecFineResNewEM", "concecFineResAgeEM"], + "concecTotalNonResEM": ["concecFineNonResNewEM", "concecFineNonResAgeEM"], + "concebcem": ["concecFineEM", "concecCoarseEM"], + "concCecpm25EM": ["concecFineEM"], } # Functions that are used to compute additional variables (i.e. one @@ -162,6 +173,17 @@ class ReadMscwCtm(GriddedReader): "ratpm10pm25": calc_ratpm10pm25, "ratpm25pm10": calc_ratpm25pm10, # "concpolyol": calc_concpolyol, + # For EC + "concecFineRes": add_dataarrays, + "concecFineNonRes": add_dataarrays, + "concecTotalRes": add_dataarrays, + "concecTotalNonRes": add_dataarrays, + "concebc": add_dataarrays, + # For EC from emission + "concecTotalResEM": add_dataarrays, + "concecTotalNonResEM": add_dataarrays, + "concebcem": add_dataarrays, + "concCecpm25EM": update_EC_units, } #: supported filename template, freq-placeholder is for frequencies diff --git a/pyaerocom/io/read_ebas.py b/pyaerocom/io/read_ebas.py index 0f07b01cb..f00541e7b 100644 --- a/pyaerocom/io/read_ebas.py +++ b/pyaerocom/io/read_ebas.py @@ -25,6 +25,7 @@ compute_wetoxs_from_concprcpoxst, compute_wetrdn_from_concprcprdn, compute_wetso4_from_concprcpso4, + compute_wetrdnpr_from_concprcprdn, concx_to_vmrx, make_proxy_drydep_from_O3, make_proxy_wetdep_from_O3, @@ -292,6 +293,8 @@ class ReadEbas(ReadUngriddedBase): "proxyweto3": ["vmro3"], "proxywetpm10": ["concprcpoxs", "pr"], "proxywetpm25": ["concprcpoxs", "pr"], + # Testing + "wetrdnpr": ["pr"], } #: Meta information supposed to be migrated to computed variables @@ -364,6 +367,8 @@ class ReadEbas(ReadUngriddedBase): "proxyweto3": make_proxy_wetdep_from_O3, "proxywetpm10": compute_wetoxs_from_concprcpoxs, "proxywetpm25": compute_wetoxs_from_concprcpoxs, + # Testing + "wetrdnpr": compute_wetrdnpr_from_concprcprdn, } #: Custom reading options for individual variables. Keys need to be valid @@ -1741,7 +1746,12 @@ def _check_keep_aux_vars(self, vars_to_retrieve): return vars_to_retrieve + add def read( - self, vars_to_retrieve=None, first_file=None, last_file=None, files=None, **constraints + self, + vars_to_retrieve=None, + first_file=None, + last_file=None, + files=None, + **constraints, ): """Method that reads list of files as instance of :class:`UngriddedData` diff --git a/tests/aeroval/test_bulkfraction_engine.py b/tests/aeroval/test_bulkfraction_engine.py new file mode 100644 index 000000000..34a517990 --- /dev/null +++ b/tests/aeroval/test_bulkfraction_engine.py @@ -0,0 +1,226 @@ +import pytest +import numpy as np +from pathlib import Path +import json + +from pyaerocom.aeroval.bulkfraction_engine import BulkFractionEngine +from pyaerocom.aeroval import ExperimentProcessor +from pyaerocom.aeroval._processing_base import HasColocator, ProcessingEngine, ExperimentOutput +from pyaerocom.aeroval import EvalSetup +from pyaerocom import ColocatedData +from tests.fixtures.aeroval import cfg_test_bulk + + +@pytest.fixture +def bulkengine_instance() -> BulkFractionEngine: + cfg = EvalSetup(**cfg_test_bulk.CFG) + bfe = BulkFractionEngine(cfg) + return bfe + + +def test___init__(): + cfg = EvalSetup(**cfg_test_bulk.CFG) + bfe = BulkFractionEngine(cfg) + assert isinstance(bfe, ProcessingEngine) + assert isinstance(bfe, HasColocator) + + +def test__get_bulk_vars(bulkengine_instance: BulkFractionEngine): + obsentry = bulkengine_instance.cfg.obs_cfg.get_entry("EBAS-m") + assert ( + bulkengine_instance._get_bulk_vars("concwetrdn", obsentry) + == obsentry.bulk_options["concwetrdn"].vars + ) + + with pytest.raises(KeyError, match="Could not find bulk vars entry"): + bulkengine_instance._get_bulk_vars("concwetrdn2", obsentry) + + +def test_get_colocators_modelexists(bulkengine_instance: BulkFractionEngine): + obsentry = bulkengine_instance.cfg.obs_cfg.get_entry("EBAS-m") + freq = obsentry.ts_type + bulk_vars = bulkengine_instance._get_bulk_vars("concwetrdn", obsentry) + cols_exist = bulkengine_instance.get_colocators( + bulk_vars, "concwetrdn", freq, "EMEP", "EBAS-m", True + ) + + assert len(cols_exist) == 2 + + assert ( + cols_exist[0][bulk_vars[0]].colocation_setup.model_use_vars[bulk_vars[0]] + == cols_exist[1][bulk_vars[1]].colocation_setup.model_use_vars[bulk_vars[1]] + ) + + +def test_get_colocators(bulkengine_instance: BulkFractionEngine): + obsentry = bulkengine_instance.cfg.obs_cfg.get_entry("EBAS-m") + freq = obsentry.ts_type + bulk_vars = bulkengine_instance._get_bulk_vars("concwetrdn", obsentry) + cols_exist = bulkengine_instance.get_colocators( + bulk_vars, "concwetrdn", freq, "EMEP", "EBAS-m", False + ) + + assert len(cols_exist) == 2 + + assert cols_exist[0][bulk_vars[0]].colocation_setup.obs_vars[0] == bulk_vars[0] + assert cols_exist[1][bulk_vars[1]].colocation_setup.obs_vars[0] == bulk_vars[1] + + +def test__run_var(bulkengine_instance: BulkFractionEngine): + obs_name = "AERONET-Sun" + model_name = "TM5-AP3-CTRL" + var_name = "fraction" + freq = "monthly" + obsentry = bulkengine_instance.cfg.obs_cfg.get_entry(obs_name) + modelentry = bulkengine_instance.cfg.model_cfg.get_entry(model_name) + bulk_vars = bulkengine_instance._get_bulk_vars(var_name, obsentry) + + col, fp = bulkengine_instance._run_var( + model_name, obs_name, var_name, bulk_vars, freq, obsentry, modelentry + ) + + assert isinstance(fp[0], str) + assert isinstance(col, ColocatedData) + assert pytest.approx(np.nanmean(col.data), rel=1e-5) == 1.0 + + +def test__combine_coldatas(bulkengine_instance: BulkFractionEngine): + obs_name = "AERONET-Sun" + model_name = "TM5-AP3-CTRL" + var_name = "fraction" + freq = "monthly" + obsentry = bulkengine_instance.cfg.obs_cfg.get_entry(obs_name) + modelentry = bulkengine_instance.cfg.model_cfg.get_entry(model_name) + bulk_vars = bulkengine_instance._get_bulk_vars(var_name, obsentry) + + num_name = bulk_vars[0] + denum_name = bulk_vars[1] + + model_exists = obsentry.bulk_options[var_name].model_exists + cols = bulkengine_instance.get_colocators( + bulk_vars, var_name, freq, model_name, obs_name, model_exists + ) + + coldatas = [] + for col in cols: + if len(list(col.keys())) != 1: + raise ValueError( + "Found more than one colocated object when trying to run bulk variable" + ) + bv = list(col.keys())[0] + coldatas.append(col[bv].run(bv)) + + obsentry2 = obsentry.model_copy() + obsentry2.bulk_options[var_name].mode = "fraction" + model_num_name, model_denum_name = bulkengine_instance._get_model_var_names( + var_name, bulk_vars, model_exists, modelentry + ) + + data1 = bulkengine_instance._combine_coldatas( + coldatas[0][model_num_name][num_name], + coldatas[1][model_denum_name][denum_name], + var_name, + obsentry2, + ) + + assert pytest.approx(np.nanmean(data1.data), rel=1e-5) == 1.0 + assert len(data1.data) == max( + len(coldatas[0][model_num_name][num_name].data), + len(coldatas[1][model_denum_name][denum_name].data), + ) + + obsentry3 = obsentry.model_copy() + obsentry3.bulk_options[var_name].mode = "product" + data2 = bulkengine_instance._combine_coldatas( + coldatas[0][model_num_name][num_name], + coldatas[1][model_denum_name][denum_name], + var_name, + obsentry3, + ) + + assert len(data2.data) == max( + len(coldatas[0][model_num_name][num_name].data), + len(coldatas[1][model_denum_name][denum_name].data), + ) + assert np.array_equal( + data2.data.data, coldatas[0][model_num_name][num_name].data.data ** 2, equal_nan=True + ) + + +def test__combine_coldatas_model_exist(bulkengine_instance: BulkFractionEngine): + obs_name = "AERONET-Sun-exist" + model_name = "TM5-AP3-CTRL" + var_name = "abs550aer" + freq = "monthly" + + obsentry = bulkengine_instance.cfg.obs_cfg.get_entry(obs_name) + modelentry = bulkengine_instance.cfg.model_cfg.get_entry(model_name) + bulk_vars = bulkengine_instance._get_bulk_vars(var_name, obsentry) + + num_name = bulk_vars[0] + denum_name = bulk_vars[1] + + model_exists = obsentry.bulk_options[var_name].model_exists + cols = bulkengine_instance.get_colocators( + bulk_vars, var_name, freq, model_name, obs_name, model_exists + ) + + coldatas = [] + for col in cols: + if len(list(col.keys())) != 1: + raise ValueError( + "Found more than one colocated object when trying to run bulk variable" + ) + bv = list(col.keys())[0] + coldatas.append(col[bv].run(bv)) + + obsentry2 = obsentry.model_copy() + obsentry2.bulk_options[var_name].mode = "fraction" + model_num_name, model_denum_name = bulkengine_instance._get_model_var_names( + var_name, bulk_vars, model_exists, modelentry + ) + + data1 = bulkengine_instance._combine_coldatas( + coldatas[0][model_num_name][num_name], + coldatas[1][model_denum_name][denum_name], + var_name, + obsentry2, + ) + + assert pytest.approx(np.nanmean(data1.data[0]), rel=1e-5) == 1.0 + assert pytest.approx(np.nanmean(data1.data[1]), rel=1e-5) != 0.0135 + + +def test_run(bulkengine_instance: BulkFractionEngine): + obs_name = "AERONET-Sun-exist" + model_name = "TM5-AP3-CTRL" + var_name = "abs550aer" + + bulkengine_instance.exp_output.delete_experiment_data(also_coldata=True) + + bulkengine_instance.run(var_name, model_name, obs_name) + + output: ExperimentOutput = bulkengine_instance.exp_output + assert Path(output.exp_dir).is_dir() + + +def test_run_cfg(): + model_name = "TM5-AP3-CTRL" + cfg = EvalSetup(**cfg_test_bulk.CFG) + proc = ExperimentProcessor(cfg) + proc.exp_output.delete_experiment_data(also_coldata=True) + proc.run(obs_name="AERONET-Sun", model_name="TM5-AP3-CTRL") + + output: ExperimentOutput = proc.exp_output + assert Path(output.exp_dir).is_dir() + + assert Path(output.experiments_file).exists() + + ts_path = Path(output.exp_dir) / "ts/ALL_AERONET-Sun-fraction_Column.json" + with open(ts_path) as f: + data = json.load(f) + m_data = data[model_name]["monthly_mod"] + o_data = data[model_name]["monthly_obs"] + + assert pytest.approx(np.nanmean(m_data), rel=1e-5) == 1.0 + assert pytest.approx(np.nanmean(o_data), rel=1e-5) == 1.0 diff --git a/tests/fixtures/aeroval/cfg_test_bulk.py b/tests/fixtures/aeroval/cfg_test_bulk.py new file mode 100644 index 000000000..6fd6498d1 --- /dev/null +++ b/tests/fixtures/aeroval/cfg_test_bulk.py @@ -0,0 +1,126 @@ +import os +from pyaerocom.variable import Variable +from pyaerocom import const + +### Register dummy fraction +variables = { + "fraction": Variable(var_name="fraction", units="1", description="fraction"), +} + +const.register_custom_variables(variables) +OBS_GROUNDBASED = { + "EBAS-m": dict( + obs_id="EBASMC", + web_interface_name="EBAS-m", + obs_vars=[ + "concwetrdn", + ], + obs_vert_type="Surface", + colocate_time=True, + ts_type="monthly", + is_bulk=True, + bulk_options={ + "concwetrdn": dict( + vars=["wetrdn", "wetrdnpr"], + model_exists=False, + mode="fraction", + units="mg N m-3", + ) + }, + ), + "AERONET-Sun": dict( + obs_id="AeronetSunV3L2Subset.daily", + obs_vars=("fraction",), + obs_vert_type="Column", + is_bulk=True, + bulk_options={ + "fraction": dict( + vars=["od550aer", "od550aer"], + model_exists=False, + mode="fraction", + units="1", + ) + }, + ts_type="monthly", + ), + "AERONET-Sun-exist": dict( + obs_id="AeronetSunV3L2Subset.daily", + obs_vars=("abs550aer",), + obs_vert_type="Column", + is_bulk=True, + bulk_options={ + "abs550aer": dict( + vars=["od550aer", "od550aer"], + model_exists=True, + mode="fraction", + units="1", + ) + }, + ts_type="monthly", + ), +} + +folder_EMEP = "/lustre/storeB/project/fou/kl/emep/ModelRuns/2022_REPORTING/TRENDS/2013/" + + +# Setup for models used in analysis +MODELS = { + "EMEP": dict( + model_id="EMEP", + model_data_dir=folder_EMEP, + gridded_reader_id={"model": "ReadMscwCtm"}, + model_ts_type_read="monthly", + ), + "TM5-AP3-CTRL": dict( + model_id="TM5-met2010_CTRL-TEST", + model_add_vars=dict(od550csaer=["od550aer"]), + model_ts_type_read="monthly", + flex_ts_type=False, + ), +} + +CFG = dict( + model_cfg=MODELS, + obs_cfg=OBS_GROUNDBASED, + json_basedir=os.path.abspath("./data"), + # coldata_basedir = os.path.abspath('../../coldata'), + coldata_basedir=os.path.abspath("./coldata"), + # io_aux_file=os.path.abspath("../../eval_py/gridded_io_aux.py"), + # if True, existing colocated data files will be deleted + reanalyse_existing=True, + only_json=False, + add_model_maps=False, + only_model_maps=False, + clear_existing_json=False, + # if True, the analysis will stop whenever an error occurs (else, errors that + # occurred will be written into the logfiles) + raise_exceptions=True, + # Regional filter for analysis + filter_name="ALL-wMOUNTAINS", + # colocation frequency (no statistics in higher resolution can be computed) + ts_type="monthly", + map_zoom="Europe", + freqs=["daily", "monthly", "yearly"], + periods=["2010"], + main_freq="monthly", + maps_freq="yearly", + zeros_to_nan=False, + colocate_time=True, + resample_how={"vmro3max": {"daily": {"hourly": "max"}}}, + obs_remove_outliers=False, + model_remove_outliers=False, + harmonise_units=True, + regions_how="country", #'default',#'country', + annual_stats_constrained=True, + proj_id="testing_bulk", + exp_id="bulktest", + exp_name="Evaluation of EMEP Bulk", + exp_descr=("Evaluation of EMEP Bulk"), + exp_pi="Daniel Heinesen", + public=True, + # directory where colocated data files are supposed to be stored + weighted_stats=True, + var_order_menu=[ + "concwetrdn", + ], +) diff --git a/tests/io/test_read_ebas.py b/tests/io/test_read_ebas.py index 508766a9d..68ccf32a3 100644 --- a/tests/io/test_read_ebas.py +++ b/tests/io/test_read_ebas.py @@ -224,6 +224,7 @@ def test_NAN_VAL(reader: ReadEbas): def test_PROVIDES_VARIABLES(reader: ReadEbas): PROVIDES_VARIABLES = { "DEFAULT", + "concebc", "concca", "concmg", "conck", @@ -368,6 +369,9 @@ def test_PROVIDES_VARIABLES(reader: ReadEbas): "concprcpoxsc", "wetoxst", "concprcpna", + "wetso4pr", + "wetrdnpr", + "wetoxspr", } assert set(reader.PROVIDES_VARIABLES) == (PROVIDES_VARIABLES) diff --git a/tests/test_varcollection.py b/tests/test_varcollection.py index 2b16d54c7..0a81637fa 100644 --- a/tests/test_varcollection.py +++ b/tests/test_varcollection.py @@ -86,7 +86,7 @@ def test_VarCollection_get_var_error(collection: VarCollection): ("*blaaaaaaa*", 0), ("dep*", 9), ("od*", 26), - ("conc*", 104), + ("conc*", 121), ], ) def test_VarCollection_find(collection: VarCollection, search_pattern: str, num: int):