From 12a1c104bb19a9602f01eae6f8958177dba85587 Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Thu, 21 Nov 2024 13:58:51 +0100 Subject: [PATCH 01/10] Set weather year for status2019 to 2011 to have a consistent weather year for the scenario path --- src/egon/data/datasets/scenario_parameters/__init__.py | 2 +- src/egon/data/datasets/scenario_parameters/parameters.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/egon/data/datasets/scenario_parameters/__init__.py b/src/egon/data/datasets/scenario_parameters/__init__.py index 3133fc287..cf019c0bc 100755 --- a/src/egon/data/datasets/scenario_parameters/__init__.py +++ b/src/egon/data/datasets/scenario_parameters/__init__.py @@ -244,7 +244,7 @@ class ScenarioParameters(Dataset): def __init__(self, dependencies): super().__init__( name="ScenarioParameters", - version="0.0.15", + version="0.0.17", dependencies=dependencies, tasks=( create_table, diff --git a/src/egon/data/datasets/scenario_parameters/parameters.py b/src/egon/data/datasets/scenario_parameters/parameters.py index 8f99cbc6e..3ccb33316 100755 --- a/src/egon/data/datasets/scenario_parameters/parameters.py +++ b/src/egon/data/datasets/scenario_parameters/parameters.py @@ -126,7 +126,7 @@ def global_settings(scenario): } elif scenario == "status2019": parameters = { - "weather_year": 2019, + "weather_year": 2011, "population_year": 2019, "fuel_costs": { # TYNDP 2020, data for 2020 (https://2020.entsos-tyndp-scenarios.eu/fuel-commodities-and-carbon-prices/) "oil": 12.9*3.6, # [EUR/MWh] From 1ed6c728ed7bd593a3986f4bd99dce42597c0bef Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Thu, 21 Nov 2024 13:59:31 +0100 Subject: [PATCH 02/10] Insert timeseries only once per weather-year --- src/egon/data/datasets/demandregio/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/egon/data/datasets/demandregio/__init__.py b/src/egon/data/datasets/demandregio/__init__.py index 7879a268f..67fa95f09 100644 --- a/src/egon/data/datasets/demandregio/__init__.py +++ b/src/egon/data/datasets/demandregio/__init__.py @@ -909,10 +909,13 @@ def timeseries_per_wz(): """ scenarios = egon.data.config.settings()["egon-data"]["--scenarios"] + year_already_in_database = [] for scn in scenarios: year = int(scenario_parameters.global_settings(scn)["weather_year"]) for sector in ["CTS", "industry"]: - insert_timeseries_per_wz(sector, int(year)) + if not year in year_already_in_database: + insert_timeseries_per_wz(sector, int(year)) + year_already_in_database.append(year) def get_cached_tables(): """Get cached demandregio tables and db-dump from former runs""" From daff790766d2f923daa1ed9d373b4714e3c40083 Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Thu, 21 Nov 2024 14:00:31 +0100 Subject: [PATCH 03/10] Add heat pumps 2019 to overall pipeline --- src/egon/data/airflow/dags/pipeline.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/egon/data/airflow/dags/pipeline.py b/src/egon/data/airflow/dags/pipeline.py index 274c1b073..ad93ad23a 100755 --- a/src/egon/data/airflow/dags/pipeline.py +++ b/src/egon/data/airflow/dags/pipeline.py @@ -53,6 +53,7 @@ HeatSupply, ) from egon.data.datasets.heat_supply.individual_heating import ( + HeatPumps2019, HeatPumps2035, HeatPumps2050, HeatPumpsPypsaEur, @@ -589,6 +590,17 @@ dependencies=[vg250, setup_etrago, create_gas_polygons] ) + # Heat pump disaggregation for status2019 + heat_pumps_2019 = HeatPumps2019( + dependencies=[ + cts_demand_buildings, + DistrictHeatingAreas, + heat_supply, + heat_time_series, + power_plants, + ] + ) + # Heat pump disaggregation for eGon2035 heat_pumps_2035 = HeatPumps2035( dependencies=[ From 1ee5d9ad78208a4458a43a5172600a9dfc6a77d1 Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Mon, 25 Nov 2024 13:17:58 +0100 Subject: [PATCH 04/10] Use population year for foreign storage unit capacities --- src/egon/data/datasets/electrical_neighbours.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/egon/data/datasets/electrical_neighbours.py b/src/egon/data/datasets/electrical_neighbours.py index 752dc959e..6fc253abc 100644 --- a/src/egon/data/datasets/electrical_neighbours.py +++ b/src/egon/data/datasets/electrical_neighbours.py @@ -1581,7 +1581,7 @@ def insert_storage_units_sq(scn_name="status2019"): None. """ - year = int(get_sector_parameters("global", scn_name)["weather_year"]) + year = int(get_sector_parameters("global", scn_name)["population_year"]) try: sto_sq = entsoe_historic_generation_capacities() except: From d5288ccd47e11a9011eb6d2172d288425a0e7339 Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Tue, 26 Nov 2024 09:06:42 +0100 Subject: [PATCH 05/10] Skip function if no CHP capacity is part of the scenario --- src/egon/data/datasets/chp/__init__.py | 4 ++++ src/egon/data/datasets/chp_etrago.py | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/src/egon/data/datasets/chp/__init__.py b/src/egon/data/datasets/chp/__init__.py index 0c20319f9..be579cbf0 100644 --- a/src/egon/data/datasets/chp/__init__.py +++ b/src/egon/data/datasets/chp/__init__.py @@ -164,6 +164,10 @@ def assign_heat_bus(): epsg=4326, ) + if chp.empty: + print(f"No CHP for district heating in scenario {scenario}") + return + # Select district heating areas and their centroid district_heating = db.select_geodataframe( f""" diff --git a/src/egon/data/datasets/chp_etrago.py b/src/egon/data/datasets/chp_etrago.py index d200a192a..5f74d6246 100644 --- a/src/egon/data/datasets/chp_etrago.py +++ b/src/egon/data/datasets/chp_etrago.py @@ -68,6 +68,10 @@ def insert_egon100re(): """ ) + if chp_dh.empty: + print("No CHP for district heating in scenario eGon100RE") + return + # Create geodataframes for gas CHP plants chp_el = link_geom_from_buses( gpd.GeoDataFrame( From fac8ce40116f800e6a6d67845e095fee70c9261f Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Tue, 26 Nov 2024 09:08:10 +0100 Subject: [PATCH 06/10] Download Biogaseinspeiseatlat-file only if it is not available --- src/egon/data/datasets/ch4_prod.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/egon/data/datasets/ch4_prod.py b/src/egon/data/datasets/ch4_prod.py index 6c67361cf..f3e0ecfba 100755 --- a/src/egon/data/datasets/ch4_prod.py +++ b/src/egon/data/datasets/ch4_prod.py @@ -198,7 +198,8 @@ def load_biogas_generators(scn_name): ) target_file = Path(".") / "datasets" / "gas_data" / basename - urlretrieve(url, target_file) + if not target_file.is_file(): + urlretrieve(url, target_file) # Read-in data from csv-file biogas_generators_list = pd.read_excel( From 8658be46beb552fa0a81bc946619864cf7df0fa4 Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Tue, 26 Nov 2024 09:10:24 +0100 Subject: [PATCH 07/10] Disaggregate PV rooftop capacities only if the are PV rooftop capacities --- src/egon/data/datasets/power_plants/pv_rooftop.py | 8 +++++++- .../datasets/power_plants/pv_rooftop_buildings.py | 11 +++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/egon/data/datasets/power_plants/pv_rooftop.py b/src/egon/data/datasets/power_plants/pv_rooftop.py index 702a52091..8e7a6a3a6 100644 --- a/src/egon/data/datasets/power_plants/pv_rooftop.py +++ b/src/egon/data/datasets/power_plants/pv_rooftop.py @@ -160,7 +160,13 @@ def pv_rooftop_per_mv_grid_and_scenario(scenario, level): WHERE carrier = 'solar_rooftop' AND scenario_name = '{scenario}' """ - ).capacity[0] + ) + + if target.empty: + print(f"No PV rooftop in scenario {scenario}") + return + else: + target = target.capacity[0] dataset = config.settings()["egon-data"]["--dataset-boundary"] diff --git a/src/egon/data/datasets/power_plants/pv_rooftop_buildings.py b/src/egon/data/datasets/power_plants/pv_rooftop_buildings.py index 785a2cc55..7b9a34074 100644 --- a/src/egon/data/datasets/power_plants/pv_rooftop_buildings.py +++ b/src/egon/data/datasets/power_plants/pv_rooftop_buildings.py @@ -2583,6 +2583,10 @@ def allocate_scenarios( """ cap_per_bus_id_df = cap_per_bus_id(scenario) + if cap_per_bus_id_df.empty: + print(f"No PV rooftop in scenario {scenario}") + return + logger.debug( f"cap_per_bus_id_df total capacity: {cap_per_bus_id_df.capacity.sum()}" ) @@ -2831,6 +2835,13 @@ def pv_rooftop_to_buildings(): <= ts ] + if cap_per_bus_id(scenario).empty: + print(f"No PV rooftop in scenario {scenario}") + EgonPowerPlantPvRoofBuildingScenario.__table__.create( + bind=engine, checkfirst=True + ) + return + logger.debug(f"Desaggregating scenario {scenario}.") ( scenario_buildings_gdf, From 0527f2521e1637ea98a2f7e4897fe31b940e35e9 Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Tue, 26 Nov 2024 09:12:29 +0100 Subject: [PATCH 08/10] Add central gas bus in Germany for eGon100RE when testmode is activated --- src/egon/data/datasets/gas_grid.py | 45 +++++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/src/egon/data/datasets/gas_grid.py b/src/egon/data/datasets/gas_grid.py index 9a0812875..52d2ecf24 100755 --- a/src/egon/data/datasets/gas_grid.py +++ b/src/egon/data/datasets/gas_grid.py @@ -308,6 +308,49 @@ def insert_gas_buses_abroad(scn_name="eGon2035"): gdf_abroad_buses.drop_duplicates( subset="country", keep="first", inplace=True ) + + if settings()["egon-data"]["--dataset-boundary"] != "Everything": + gdf_abroad_buses_insert = pd.DataFrame( + index=[gdf_abroad_buses.index.max() + 1], + data={ + "scn_name": scn_name, + "bus_id": (db.next_etrago_id("bus") + len(gdf_abroad_buses) + 1), + "x": 10.4234469, + "y": 51.0834196, + "country": "DE", + "carrier": gas_carrier, + }, + ) + + gdf_abroad_buses_insert = geopandas.GeoDataFrame( + gdf_abroad_buses_insert, + geometry=geopandas.points_from_xy( + gdf_abroad_buses_insert["x"], gdf_abroad_buses_insert["y"] + ), + ) + gdf_abroad_buses_insert = gdf_abroad_buses_insert.rename( + columns={"geometry": "geom"} + ).set_geometry("geom", crs=4326) + + # Insert to db + print(gdf_abroad_buses_insert) + gdf_abroad_buses_insert.to_postgis( + "egon_etrago_bus", + engine, + schema="grid", + index=False, + if_exists="append", + dtype={"geom": Geometry()}, + ) + + gdf_abroad_buses = pd.concat( + [ + gdf_abroad_buses, + gdf_abroad_buses_insert + ], + ignore_index=True, + ) + return gdf_abroad_buses else: @@ -1031,7 +1074,7 @@ class GasNodesAndPipes(Dataset): #: name: str = "GasNodesAndPipes" #: - version: str = "0.0.10" + version: str = "0.0.11" tasks = (insert_gas_data_status2019, insert_gas_data) From 21f6e708b91137f575e6ca39c32f77bbff8c3f5a Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Tue, 26 Nov 2024 09:13:56 +0100 Subject: [PATCH 09/10] Do not execute power_to_h2 and h2_to_ch4 for status2019 --- src/egon/data/datasets/hydrogen_etrago/h2_to_ch4.py | 3 +++ src/egon/data/datasets/hydrogen_etrago/power_to_h2.py | 3 +++ 2 files changed, 6 insertions(+) diff --git a/src/egon/data/datasets/hydrogen_etrago/h2_to_ch4.py b/src/egon/data/datasets/hydrogen_etrago/h2_to_ch4.py index 4e8591e14..fb9ff021c 100755 --- a/src/egon/data/datasets/hydrogen_etrago/h2_to_ch4.py +++ b/src/egon/data/datasets/hydrogen_etrago/h2_to_ch4.py @@ -40,6 +40,9 @@ def insert_h2_to_ch4_to_h2(): """ scenarios = config.settings()["egon-data"]["--scenarios"] + if "status2019" in scenarios: + scenarios.remove("status2019") + for scn_name in scenarios: # Connect to local database engine = db.engine() diff --git a/src/egon/data/datasets/hydrogen_etrago/power_to_h2.py b/src/egon/data/datasets/hydrogen_etrago/power_to_h2.py index 77501773e..9d1ab4303 100755 --- a/src/egon/data/datasets/hydrogen_etrago/power_to_h2.py +++ b/src/egon/data/datasets/hydrogen_etrago/power_to_h2.py @@ -43,6 +43,9 @@ def insert_power_to_h2_to_power(): """ scenarios = config.settings()["egon-data"]["--scenarios"] + if "status2019" in scenarios: + scenarios.remove("status2019") + for scn_name in scenarios: # Connect to local database From 693041048916e118d56350590650ef4066d83549 Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Tue, 26 Nov 2024 09:34:06 +0100 Subject: [PATCH 10/10] Do not import status2019 from backup if it was created within the pipeline --- .../data/datasets/scenario_path/import_status2019.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/egon/data/datasets/scenario_path/import_status2019.py b/src/egon/data/datasets/scenario_path/import_status2019.py index 126b24c82..509fdcb25 100644 --- a/src/egon/data/datasets/scenario_path/import_status2019.py +++ b/src/egon/data/datasets/scenario_path/import_status2019.py @@ -24,6 +24,12 @@ def download_status2019(): None. """ + + if "status2019" in config.settings()["egon-data"]["--scenarios"]: + print("Scenario status2019 was created within the pipeline. " + "It is not imported from zenodo.") + return + # Get parameters from config and set download URL url = sources["url_status2019"] status2019_path = Path(".") / "PoWerD_status2019.backup" @@ -43,6 +49,12 @@ def import_scn_status2019(): *No parameters required """ + + if "status2019" in config.settings()["egon-data"]["--scenarios"]: + print("Scenario status2019 was created within the pipeline. " + "It is not imported from zenodo.") + return + # Connect to the data base con = db.engine()