From 186f1b3571c53c51526ea1814de1da31094ba0d6 Mon Sep 17 00:00:00 2001 From: romainsacchi Date: Sat, 22 Jul 2023 13:37:59 +0200 Subject: [PATCH] EODC --- premise/activity_maps.py | 130 ++++++++--------- premise/ecoinvent_modification.py | 35 ++--- premise/electricity.py | 36 +++++ .../biomass_variables.yaml | 2 + .../electricity_variables.yaml | 131 +++++++++++++----- 5 files changed, 210 insertions(+), 124 deletions(-) diff --git a/premise/activity_maps.py b/premise/activity_maps.py index 72bc6122..938eab7c 100644 --- a/premise/activity_maps.py +++ b/premise/activity_maps.py @@ -41,6 +41,59 @@ def get_mapping(filepath: Path, var: str) -> dict: return mapping +def act_fltr( + database: List[dict], + fltr: Union[str, List[str]] = None, + mask: Union[str, List[str]] = None, +) -> List[dict]: + """Filter `database` for activities matching field contents given by `fltr` excluding strings in `mask`. + `fltr`: string, list of strings or dictionary. + If a string is provided, it is used to match the name field from the start (*startswith*). + If a list is provided, all strings in the lists are used and results are joined (*or*). + A dict can be given in the form : to filter for in . + `mask`: used in the same way as `fltr`, but filters add up with each other (*and*). + `filter_exact` and `mask_exact`: boolean, set `True` to only allow for exact matches. + + :param database: A lice cycle inventory database + :type database: brightway2 database object + :param fltr: value(s) to filter with. + :type fltr: Union[str, lst, dict] + :param mask: value(s) to filter with. + :type mask: Union[str, lst, dict] + :return: list of activity data set names + :rtype: list + + """ + if fltr is None: + fltr = {} + if mask is None: + mask = {} + + # default field is name + if isinstance(fltr, (list, str)): + fltr = {"name": fltr} + if isinstance(mask, (list, str)): + mask = {"name": mask} + + assert len(fltr) > 0, "Filter dict must not be empty." + + # find `act` in `database` that match `fltr` + # and do not match `mask` + filters = [] + for field, value in fltr.items(): + if isinstance(value, list): + filters.extend([ws.either(*[ws.contains(field, v) for v in value])]) + else: + filters.append(ws.contains(field, value)) + + for field, value in mask.items(): + if isinstance(value, list): + filters.extend([ws.exclude(ws.contains(field, v)) for v in value]) + else: + filters.append(ws.exclude(ws.contains(field, value))) + + return list(ws.get_many(database, *filters)) + class InventorySet: """ @@ -181,84 +234,13 @@ def generate_material_map(self) -> dict: """ return self.generate_sets_from_filters(self.materials_filters) - @staticmethod - def act_fltr( - database: List[dict], - fltr: Union[str, List[str]] = None, - mask: Union[str, List[str]] = None, - filter_exact: bool = False, - mask_exact: bool = False, - ) -> List[dict]: - """Filter `database` for activities matching field contents given by `fltr` excluding strings in `mask`. - `fltr`: string, list of strings or dictionary. - If a string is provided, it is used to match the name field from the start (*startswith*). - If a list is provided, all strings in the lists are used and results are joined (*or*). - A dict can be given in the form : to filter for in . - `mask`: used in the same way as `fltr`, but filters add up with each other (*and*). - `filter_exact` and `mask_exact`: boolean, set `True` to only allow for exact matches. - - :param database: A lice cycle inventory database - :type database: brightway2 database object - :param fltr: value(s) to filter with. - :type fltr: Union[str, lst, dict] - :param mask: value(s) to filter with. - :type mask: Union[str, lst, dict] - :param filter_exact: requires exact match when true. - :type filter_exact: bool - :param mask_exact: requires exact match when true. - :type mask_exact: bool - :return: list of activity data set names - :rtype: list - - """ - if fltr is None: - fltr = {} - if mask is None: - mask = {} - result = [] - - # default field is name - if isinstance(fltr, (list, str)): - fltr = {"name": fltr} - if isinstance(mask, (list, str)): - mask = {"name": mask} - - def like(item_a, item_b): - if filter_exact: - return item_a.lower() == item_b.lower() - return item_a.lower().startswith(item_b.lower()) - - def notlike(item_a, item_b): - if mask_exact: - return item_a.lower() != item_b.lower() - return item_b.lower() not in item_a.lower() - - assert len(fltr) > 0, "Filter dict must not be empty." - - # find `act` in `database` that match `fltr` - # and do not match `mask` - filters = [] - for field, value in fltr.items(): - if isinstance(value, list): - filters.extend([ws.either(*[ws.contains(field, v) for v in value])]) - else: - filters.append(ws.contains(field, value)) - - for field, value in mask.items(): - if isinstance(value, list): - filters.extend([ws.exclude(ws.contains(field, v)) for v in value]) - else: - filters.append(ws.exclude(ws.contains(field, value))) - - return list(ws.get_many(database, *filters)) - def generate_sets_from_filters(self, filtr: dict, database=None) -> dict: """ Generate a dictionary with sets of activity names for technologies from the filter specifications. - :param filtr: - :func:`activity_maps.InventorySet.act_fltr`. + :param filtr: + :func:`activity_maps.InventorySet.act_fltr`. :return: dictionary with the same keys as provided in filter and a set of activity data set names as values. :rtype: dict @@ -266,5 +248,5 @@ def generate_sets_from_filters(self, filtr: dict, database=None) -> dict: database = database or self.database - techs = {tech: self.act_fltr(database, **fltr) for tech, fltr in filtr.items()} + techs = {tech: act_fltr(database, **fltr) for tech, fltr in filtr.items()} return {tech: {act["name"] for act in actlst} for tech, actlst in techs.items()} diff --git a/premise/ecoinvent_modification.py b/premise/ecoinvent_modification.py index 97f608f8..72b72436 100644 --- a/premise/ecoinvent_modification.py +++ b/premise/ecoinvent_modification.py @@ -765,31 +765,32 @@ def update_electricity(self) -> None: use_absolute_efficiency=self.use_absolute_efficiency, ) - electricity.adjust_coal_power_plant_emissions() + electricity.create_missing_power_plant_datasets() + #electricity.adjust_coal_power_plant_emissions() # datasets in 3.9 have been updated - if self.version not in ["3.9", "3.9.1"]: - electricity.update_ng_production_ds() + #if self.version not in ["3.9", "3.9.1"]: + # electricity.update_ng_production_ds() - electricity.update_efficiency_of_solar_pv() + #electricity.update_efficiency_of_solar_pv() - if scenario["iam data"].biomass_markets is not None: - electricity.create_biomass_markets() + #if scenario["iam data"].biomass_markets is not None: + # electricity.create_biomass_markets() - electricity.create_region_specific_power_plants() + #electricity.create_region_specific_power_plants() - if scenario["iam data"].electricity_markets is not None: - electricity.update_electricity_markets() - else: - print("No electricity markets found in IAM data. Skipping.") + #if scenario["iam data"].electricity_markets is not None: + # electricity.update_electricity_markets() + #else: + # print("No electricity markets found in IAM data. Skipping.") - if scenario["iam data"].electricity_efficiencies is not None: - electricity.update_electricity_efficiency() - else: - print("No electricity efficiencies found in IAM data. Skipping.") + #if scenario["iam data"].electricity_efficiencies is not None: + # electricity.update_electricity_efficiency() + #else: + # print("No electricity efficiencies found in IAM data. Skipping.") - scenario["database"] = electricity.database - self.modified_datasets = electricity.modified_datasets + #scenario["database"] = electricity.database + #self.modified_datasets = electricity.modified_datasets def update_dac(self) -> None: """ diff --git a/premise/electricity.py b/premise/electricity.py index 5ee4b3f0..7217b381 100644 --- a/premise/electricity.py +++ b/premise/electricity.py @@ -36,9 +36,11 @@ ws, ) from .utils import DATA_DIR, eidb_label, get_efficiency_solar_photovoltaics +from .activity_maps import act_fltr LOSS_PER_COUNTRY = DATA_DIR / "electricity" / "losses_per_country.csv" IAM_BIOMASS_VARS = VARIABLES_DIR / "biomass_variables.yaml" +POWERPLANT_TECHS = VARIABLES_DIR / "electricity_variables.yaml" LOG_CONFIG = DATA_DIR / "utils" / "logging" / "logconfig.yaml" # directory for log files @@ -55,6 +57,19 @@ logger = logging.getLogger("electricity") +def load_electricity_variables() -> dict: + """ + Load the electricity variables from a YAML file. + :return: a dictionary with the electricity variables + :rtype: dict + """ + + with open(POWERPLANT_TECHS, "r", encoding="utf-8") as stream: + techs = yaml.full_load(stream) + + return techs + + def get_losses_per_country_dict() -> Dict[str, Dict[str, float]]: """ Create a dictionary with ISO country codes as keys and loss ratios as values. @@ -1926,6 +1941,27 @@ def adjust_coal_power_plant_emissions(self) -> None: # self.write_log(dataset=dataset, status="updated") + def create_missing_power_plant_datasets(self) -> None: + """ + Create missing power plant datasets. + We use proxy datasets, copy them and rename them. + """ + for tech, vars in load_electricity_variables().items(): + if not vars.get("exists in database", True): + datasets = act_fltr( + self.database, + vars["proxy"]["filter"], + vars["proxy"].get("mask", {}), + ) + + print(tech, len(datasets)) + + + + + + + def update_electricity_markets(self) -> None: """ Delete electricity markets. Create high, medium and low voltage market groups for electricity. diff --git a/premise/iam_variables_mapping/biomass_variables.yaml b/premise/iam_variables_mapping/biomass_variables.yaml index 7f8b67da..3fa8acbc 100644 --- a/premise/iam_variables_mapping/biomass_variables.yaml +++ b/premise/iam_variables_mapping/biomass_variables.yaml @@ -17,7 +17,9 @@ biomass wood - purpose grown: message: Primary Energy|Biomass|Fuelwood ecoinvent_aliases: name: + - market for wood chips reference product: + - wood chips biomass - residual: iam_aliases: message: Primary Energy|Biomass|Residues diff --git a/premise/iam_variables_mapping/electricity_variables.yaml b/premise/iam_variables_mapping/electricity_variables.yaml index c61daaf3..09255708 100644 --- a/premise/iam_variables_mapping/electricity_variables.yaml +++ b/premise/iam_variables_mapping/electricity_variables.yaml @@ -23,18 +23,19 @@ Biomass CHP: ecoinvent_fuel_aliases: fltr: - market for wood chips, wet, measured as dry mass + Biomass CHP CCS: iam_aliases: image: Secondary Energy|Electricity|Biomass|w/ CCS|2 eff_aliases: image: Efficiency|Electricity|Biomass|w/ CCS|2 - ecoinvent_aliases: fltr: - electricity production, at co-generation wood-fired power plant, post, pipeline 200km, storage 1000m ecoinvent_fuel_aliases: fltr: - heat and power co-generation, wood chips, 6667 kW + Biomass ST: iam_aliases: message: Secondary Energy|Electricity|Biomass|w/o CCS|2 @@ -42,7 +43,6 @@ Biomass ST: eff_aliases: message: Efficiency|Electricity|Biomass|w/o CCS|2 image: Efficiency|Electricity|Biomass|w/o CCS|1 - ecoinvent_aliases: fltr: - electricity production, at wood burning power plant @@ -50,7 +50,6 @@ Biomass ST: name: pipeline ecoinvent_fuel_aliases: fltr: - - market for wood chips, wet, measured as dry mass - market for biomass, used as fuel @@ -66,11 +65,11 @@ Biomass IGCC CCS: ecoinvent_aliases: fltr: - electricity production, at biomass-fired IGCC power plant, pre, pipeline 200km, storage 1000m - ecoinvent_fuel_aliases: fltr: - market for wood chips, wet, measured as dry mass - market for biomass, used as fuel + Biomass IGCC: iam_aliases: message: Secondary Energy|Electricity|Biomass|w/o CCS|1 @@ -89,6 +88,7 @@ Biomass IGCC: fltr: - market for wood chips, wet, measured as dry mass - market for biomass, used as fuel + Coal PC: iam_aliases: message: Secondary Energy|Electricity|Coal|w/o CCS|3 @@ -98,13 +98,14 @@ Coal PC: message: Efficiency|Electricity|Coal|w/o CCS|3 remind: Tech|Electricity|Coal|Pulverised Coal w/o CC|Efficiency image: Efficiency|Electricity|Coal|w/o CCS|1 - ecoinvent_aliases: fltr: - electricity production, hard coal - electricity production, lignite mask: - name: mine + name: + - mine + - supercritical ecoinvent_fuel_aliases: fltr: - market for hard coal @@ -115,6 +116,7 @@ Coal PC: - plant - briquettes - ash + Coal IGCC: iam_aliases: message: Secondary Energy|Electricity|Coal|w/o CCS|1 @@ -124,17 +126,14 @@ Coal IGCC: message: Efficiency|Electricity|Coal|w/o CCS|1 remind: Tech|Electricity|Coal|Gasification Combined Cycle w/o CC|Efficiency image: Efficiency|Electricity|Coal|w/o CCS|2 - ecoinvent_aliases: fltr: - electricity production, at hard coal-fired IGCC power plant - electricity production, at lignite-fired IGCC power plant mask: name: pipeline - ecoinvent_fuel_aliases: fltr: - - market for hard coal - lignite mine operation mask: @@ -151,7 +150,6 @@ Coal PC CCS: eff_aliases: message: Efficiency|Electricity|Coal|w/ CCS|2 remind: Tech|Electricity|Coal|Pulverised Coal w/ CC|Efficiency - ecoinvent_aliases: fltr: - electricity production, at hard coal-fired power plant, oxy, pipeline 200km, storage 1000m @@ -160,7 +158,6 @@ Coal PC CCS: - electricity production, at lignite-fired power plant, post, pipeline 200km, storage 1000m ecoinvent_fuel_aliases: fltr: - - market for hard coal - lignite mine operation mask: @@ -179,15 +176,12 @@ Coal IGCC CCS: message: Efficiency|Electricity|Coal|w/ CCS|1 remind: Tech|Electricity|Coal|Gasification Combined Cycle w/ CC|Efficiency image: Efficiency|Electricity|Coal|w/ CCS|1 - ecoinvent_aliases: fltr: - electricity production, at hard coal-fired IGCC power plant, pre, pipeline 200km, storage 1000m - electricity production, at lignite-fired IGCC power plant, pre, pipeline 200km, storage 1000m - ecoinvent_fuel_aliases: fltr: - - market for hard coal - lignite mine operation mask: @@ -204,7 +198,6 @@ Coal CHP: eff_aliases: remind: Tech|Electricity|Coal|Combined Heat and Power w/o CC|Efficiency image: Efficiency|Electricity|Coal|w/o CCS|3 - ecoinvent_aliases: fltr: - heat and power co-generation, hard coal @@ -221,29 +214,49 @@ Coal CHP: - plant - briquettes - ash + Coal CHP CCS: iam_aliases: image: Secondary Energy|Electricity|Coal|w/ CCS|2 eff_aliases: image: Efficiency|Electricity|Coal|w/ CCS|2 - ecoinvent_aliases: fltr: - electricity production, at co-generation hard coal-fired power plant, post, pipeline 200km, storage 1000m - ecoinvent_fuel_aliases: fltr: - heat and power co-generation, hard coal + Coal SC: # super critical steam cycle iam_aliases: message: Secondary Energy|Electricity|Coal|w/o CCS|2 eff_aliases: message: Efficiency|Electricity|Coal|w/o CCS|2 + ecoinvent_aliases: + fltr: + - electricity production, hard coal, supercritical + ecoinvent_fuel_aliases: + fltr: + - market for hard coal + Coal PCU: # sub critical steam cycle unfiltered iam_aliases: message: Secondary Energy|Electricity|Coal|w/o CCS|4 eff_aliases: message: Efficiency|Electricity|Coal|w/o CCS|4 + ecoinvent_aliases: + fltr: + - electricity production, hard coal, subcritical + exists in database: False + proxy: + fltr: + - electricity production, hard coal + mask: + name: + - mine + - supercritical + new name: electricity production, hard coal, subcritical + Gas OC: iam_aliases: @@ -254,7 +267,6 @@ Gas OC: message: Efficiency|Electricity|Gas|w/o CCS|3 remind: Tech|Electricity|Gas|Gas Turbine|Efficiency image: Efficiency|Electricity|Gas|w/o CCS|1 - ecoinvent_fuel_aliases: fltr: - market for natural gas, high pressure @@ -269,10 +281,10 @@ Gas OC: - burned - vented - vehicle - ecoinvent_aliases: fltr: - electricity production, natural gas, conventional power plant + Gas CC: iam_aliases: message: Secondary Energy|Electricity|Gas|w/o CCS|1 @@ -282,7 +294,6 @@ Gas CC: message: Efficiency|Electricity|Gas|w/o CCS|1 remind: Tech|Electricity|Gas|Combined Cycle w/o CC|Efficiency image: Efficiency|Electricity|Gas|w/o CCS|2 - ecoinvent_aliases: fltr: - electricity production, natural gas, combined cycle power plant @@ -299,6 +310,7 @@ Gas CC: - burned - vented - vehicle + Gas CHP: iam_aliases: remind: SE|Electricity|Gas|++|Combined Heat and Power w/o CC @@ -306,7 +318,6 @@ Gas CHP: eff_aliases: remind: Tech|Electricity|Gas|Combined Heat and Power w/o CC|Efficiency image: Efficiency|Electricity|Gas|w/o CCS|3 - ecoinvent_aliases: fltr: - heat and power co-generation, natural gas, combined cycle power plant, 400MW electrical @@ -333,11 +344,9 @@ Gas CHP CCS: image: Secondary Energy|Electricity|Gas|w/ CCS|2 eff_aliases: image: Efficiency|Electricity|Gas|w/ CCS|2 - ecoinvent_aliases: fltr: - electricity production, at co-generation natural gas-fired power plant, post, pipeline 200km, storage 1000m - ecoinvent_fuel_aliases: fltr: - heat and power co-generation, natural gas, conventional power plant, 100MW electrical @@ -351,11 +360,9 @@ Gas CC CCS: message: Efficiency|Electricity|Gas|w/ CCS|1 remind: Tech|Electricity|Gas|Combined Cycle w/ CC|Efficiency image: Efficiency|Electricity|Gas|w/ CCS|1 - ecoinvent_aliases: fltr: - electricity production, at natural gas-fired combined cycle power plant, post, pipeline 200km, storage 1000m - ecoinvent_fuel_aliases: fltr: - market group for natural gas @@ -370,11 +377,18 @@ Gas ST: # ["Note": "Steam cycle sub cricitcal"] message: Efficiency|Electricity|Gas|w/o CCS|2 remind: Tech|Electricity|Gas|Gas Turbine|Efficiency image: Efficiency|Electricity|Gas|w/o CCS|1 - ecoinvent_fuel_aliases: + ecoinvent_aliases: + fltr: + - electricity production, natural gas, subcritical, steam cycle + ecoinvent_fuel_aliases: fltr: - market group for natural gas - market for natural gas - + exists in database: False + proxy: + fltr: + - electricity production, natural gas, conventional power plant + new name: electricity production, natural gas, subcritical, steam cycle Geothermal: iam_aliases: @@ -384,6 +398,7 @@ Geothermal: ecoinvent_aliases: fltr: - electricity production, deep geothermal + Hydro: iam_aliases: message: Secondary Energy|Electricity|Hydro @@ -395,6 +410,7 @@ Hydro: - electricity production, hydro, run-of-river mask: name: renewable + Nuclear: iam_aliases: message: Secondary Energy|Electricity|Nuclear @@ -428,6 +444,7 @@ Nuclear_EPR: - market for nuclear fuel element, for pressure water reactor - market for nuclear fuel element, for boiling water reactor - market for uranium hexafluoride + Nuclear_SMR: ecoinvent_aliases: fltr: @@ -447,7 +464,6 @@ Oil ST: eff_aliases: remind: Tech|Electricity|Oil|DOT|Efficiency image: Efficiency|Electricity|Oil|w/o CCS|1 - ecoinvent_aliases: fltr: - electricity production, oil @@ -460,6 +476,7 @@ Oil ST: - market group for heavy fuel oil mask: name: burned + Oil CC: iam_aliases: message: Secondary Energy|Electricity|Oil|w/o CCS|1 @@ -467,7 +484,6 @@ Oil CC: eff_aliases: message: Efficiency|Electricity|Oil|w/o CCS|1 image: Efficiency|Electricity|Oil|w/o CCS|2 - ecoinvent_aliases: fltr: - electricity production, oil @@ -480,12 +496,12 @@ Oil CC: - market group for heavy fuel oil mask: name: burned + Oil CC CCS: iam_aliases: image: Secondary Energy|Electricity|Oil|w/ CCS|1 eff_aliases: image: Efficiency|Electricity|Oil|w/ CCS|1 - ecoinvent_aliases: fltr: - electricity production, at co-generation oil-fired power plant, post, pipeline 200km, storage 1000m @@ -496,12 +512,12 @@ Oil CC CCS: - heat and power co-generation, oil mask: name: burned + Oil CHP: iam_aliases: image: Secondary Energy|Electricity|Oil|w/o CCS|3 eff_aliases: image: Efficiency|Electricity|Oil|w/o CCS|3 - ecoinvent_aliases: fltr: - heat and power co-generation, oil @@ -514,16 +530,15 @@ Oil CHP: - market group for heavy fuel oil mask: name: burned + Oil CHP CCS: iam_aliases: image: Secondary Energy|Electricity|Oil|w/ CCS|2 eff_aliases: image: Efficiency|Electricity|Oil|w/ CCS|2 - ecoinvent_aliases: fltr: - electricity production, at co-generation oil-fired power plant, post, pipeline 200km, storage 1000m - mask: name: aluminium ecoinvent_fuel_aliases: @@ -531,16 +546,52 @@ Oil CHP CCS: - heat and power co-generation, oil mask: name: burned + Loil ST: # ["Note": "Steam cycle light oil"] iam_aliases: message: Secondary Energy|Electricity|Oil|w/o CCS|2 eff_aliases: message: Efficiency|Electricity|Oil|w/o CCS|2 + ecoinvent_aliases: + fltr: + - electricity production, light fuel oil, steam cycle + ecoinvent_fuel_aliases: + fltr: + - market for heavy fuel oil + - market group for heavy fuel oil + mask: + name: burned + exists in database: False + proxy: + fltr: + - electricity production, oil + mask: + name: aluminium + reference product: heat + new name: electricity production, light fuel oil, steam cycle + Foil ST: # ["Note": "Steam cycle fuel oil"] iam_aliases: message: Secondary Energy|Electricity|Oil|w/o CCS|3 eff_aliases: message: Efficiency|Electricity|Oil|w/o CCS|3 + ecoinvent_aliases: + fltr: + - electricity production, fuel oil, steam cycle + ecoinvent_fuel_aliases: + fltr: + - market for heavy fuel oil + - market group for heavy fuel oil + mask: + name: burned + exists in database: False + proxy: + fltr: + - electricity production, oil + mask: + name: aluminium + reference product: heat + new name: electricity production, fuel oil, steam cycle Solar CSP: iam_aliases: @@ -551,9 +602,19 @@ Solar CSP: fltr: - electricity production, solar thermal parabolic trough, 50 MW - electricity production, solar tower power plant, 20 MW + Solar CSP 3x: iam_aliases: message: Secondary Energy|Electricity|Solar|CSP|2 + ecoinvent_aliases: + fltr: + - electricity production, solar thermal parabolic trough, 150 MW + exists in database: False + proxy: + fltr: + - electricity production, solar thermal parabolic trough, 50 MW + new name: electricity production, solar thermal parabolic trough, 150 MW + Solar PV Centralized: iam_aliases: message: Secondary Energy|Electricity|Solar|PV @@ -562,12 +623,14 @@ Solar PV Centralized: ecoinvent_aliases: fltr: - electricity production, photovoltaic, commercial + Solar PV Residential: iam_aliases: image: Secondary Energy|Electricity|Solar|PV|2 ecoinvent_aliases: fltr: - electricity production, photovoltaic, residential + Wind Onshore: iam_aliases: message: Secondary Energy|Electricity|Wind|Onshore @@ -582,6 +645,7 @@ Wind Onshore: name: - label-certified - renewable + Wind Offshore: iam_aliases: message: Secondary Energy|Electricity|Wind|Offshore @@ -594,6 +658,7 @@ Wind Offshore: name: - label-certified - renewable + Wave: ecoinvent_aliases: fltr: