Merge remote-tracking branch 'origin/dev' into features/insert_h2_grid

CHANGELOG.rst

@@ -738,6 +738,9 @@ Bug Fixes
   `#179 <https://github.com/openego/powerd-data/issues/179>`_
 * Fix depricated python-operator import
   `#183 <https://github.com/openego/powerd-data/issues/183>`_
+* Fix incorrect YAML serialization of the --scenarios CLI
+  parameter that caused broken DAGs in the pipeline.
+  `#343 <https://github.com/openego/powerd-data/issues/343>`_
 
 
 .. _PR #692: https://github.com/openego/eGon-data/pull/692

src/egon/data/airflow/dags/pipeline.py

@@ -53,6 +53,7 @@
     HeatSupply,
 )
 from egon.data.datasets.heat_supply.individual_heating import (
+    HeatPumps2019,
     HeatPumps2035,
     HeatPumps2050,
     HeatPumpsPypsaEur,
@@ -98,6 +99,7 @@
 from egon.data.datasets.zensus import ZensusMiscellaneous, ZensusPopulation
 from egon.data.datasets.zensus_mv_grid_districts import ZensusMvGridDistricts
 from egon.data.datasets.zensus_vg250 import ZensusVg250
+from egon.data.datasets.scenario_path import CreateIntermediateScenarios
 
 # Set number of threads used by numpy and pandas
 set_numexpr_threads()
@@ -588,6 +590,17 @@
         dependencies=[vg250, setup_etrago, create_gas_polygons]
     )
 
+    # Heat pump disaggregation for status2019
+    heat_pumps_2019 = HeatPumps2019(
+        dependencies=[
+            cts_demand_buildings,
+            DistrictHeatingAreas,
+            heat_supply,
+            heat_time_series,
+            power_plants,
+        ]
+    )
+
     # Heat pump disaggregation for eGon2035
     heat_pumps_2035 = HeatPumps2035(
         dependencies=[
@@ -672,6 +685,18 @@
         ]
     )
 
+    # Create intermediate scenarios based on status2019 and eGon100RE
+    create_intemediate_scenarios = CreateIntermediateScenarios(
+        dependencies=[
+            storage_etrago,
+            hts_etrago_table,
+            fill_etrago_generators,
+            household_electricity_demand_annual,
+            cts_demand_buildings,
+            emobility_mit,
+        ]
+    )
+
     # ########## Keep this dataset at the end
     # Sanity Checks
     sanity_checks = SanityChecks(

src/egon/data/airflow/dags/pipeline_status_quo.py

@@ -335,7 +335,7 @@
 
     # Deal with electrical neighbours
     foreign_lines = ElectricalNeighbours(
-        dependencies=[prepare_pypsa_eur, tyndp_data]
+        dependencies=[prepare_pypsa_eur, tyndp_data, osmtgmod, fix_subnetworks]
     )
 
     # Import NEP (Netzentwicklungsplan) data

src/egon/data/cli.py

@@ -179,6 +179,7 @@
     default=["status2019", "eGon2035"],
     metavar="SCENARIOS",
     help=("List of scenario names for which a data model shall be created."),
+    multiple=True,
     show_default=True,
 )
 @click.option(

src/egon/data/datasets.yml

@@ -1281,3 +1281,7 @@ home_batteries:
     home_batteries:
       schema: 'supply'
       table: 'egon_home_batteries'
+
+scenario_path:
+  sources:
+    url_status2019: 'https://zenodo.org/records/13865306/files/PoWerD_status2019_v2.backup'

src/egon/data/datasets/ch4_prod.py

@@ -198,7 +198,8 @@ def load_biogas_generators(scn_name):
     )
     target_file = Path(".") / "datasets" / "gas_data" / basename
 
-    urlretrieve(url, target_file)
+    if not target_file.is_file():
+        urlretrieve(url, target_file)
 
     # Read-in data from csv-file
     biogas_generators_list = pd.read_excel(

src/egon/data/datasets/chp/__init__.py

@@ -164,6 +164,10 @@ def assign_heat_bus():
             epsg=4326,
         )
 
+        if chp.empty:
+            print(f"No CHP for district heating in scenario {scenario}")
+            return
+
         # Select district heating areas and their centroid
         district_heating = db.select_geodataframe(
             f"""

src/egon/data/datasets/chp_etrago.py

@@ -68,6 +68,10 @@ def insert_egon100re():
         """
     )
 
+    if chp_dh.empty:
+        print("No CHP for district heating in scenario eGon100RE")
+        return
+
     # Create geodataframes for gas CHP plants
     chp_el = link_geom_from_buses(
         gpd.GeoDataFrame(

src/egon/data/datasets/demandregio/__init__.py

@@ -925,10 +925,13 @@ def timeseries_per_wz():
 
     """
     scenarios = egon.data.config.settings()["egon-data"]["--scenarios"]
+    year_already_in_database = []
     for scn in scenarios:
         year = int(scenario_parameters.global_settings(scn)["weather_year"])
         for sector in ["CTS", "industry"]:
-            insert_timeseries_per_wz(sector, int(year))
+            if not year in year_already_in_database:
+                insert_timeseries_per_wz(sector, int(year))
+        year_already_in_database.append(year)
 
 def get_cached_tables():
     """Get cached demandregio tables and db-dump from former runs"""

src/egon/data/datasets/electrical_neighbours.py

@@ -538,6 +538,7 @@ def cross_border_lines(scenario, sources, targets, central_buses):
 
     new_lines = new_lines[new_lines.bus0 != new_lines.bus1]
 
+    new_lines["cables"] = new_lines["cables"].apply(int)
 
     # Insert lines to the database
     new_lines.to_postgis(
@@ -1581,7 +1582,7 @@ def insert_storage_units_sq(scn_name="status2019"):
     None.
 
     """
-    year = int(get_sector_parameters("global", scn_name)["weather_year"])
+    year = int(get_sector_parameters("global", scn_name)["population_year"])
     try:
         sto_sq = entsoe_historic_generation_capacities()
     except:
@@ -1683,7 +1684,8 @@ def insert_storage_units_sq(scn_name="status2019"):
     )
 
     # Select year of interest
-    bat_sq = bat_sq[[year]].rename(columns={year: "p_nom"})
+    bat_sq = bat_sq[[str(year)]]
+    bat_sq.rename(columns={str(year): "p_nom"}, inplace= True)
 
     # Add missing information suitable for eTraGo selected from scenario_parameter table
     parameters_batteries = get_sector_parameters(
@@ -2020,7 +2022,7 @@ class ElectricalNeighbours(Dataset):
     def __init__(self, dependencies):
         super().__init__(
             name="ElectricalNeighbours",
-            version="0.0.10",
+            version="0.0.11",
             dependencies=dependencies,
             tasks=tasks,
         )

src/egon/data/datasets/gas_grid.py

@@ -308,6 +308,49 @@ def insert_gas_buses_abroad(scn_name="eGon2035"):
         gdf_abroad_buses.drop_duplicates(
             subset="country", keep="first", inplace=True
         )
+
+        if settings()["egon-data"]["--dataset-boundary"] != "Everything":
+            gdf_abroad_buses_insert = pd.DataFrame(
+                        index=[gdf_abroad_buses.index.max() + 1],
+                        data={
+                            "scn_name": scn_name,
+                            "bus_id": (db.next_etrago_id("bus") + len(gdf_abroad_buses) + 1),
+                            "x": 10.4234469,
+                            "y": 51.0834196,
+                            "country": "DE",
+                            "carrier": gas_carrier,
+                        },
+                    )
+
+            gdf_abroad_buses_insert = geopandas.GeoDataFrame(
+                gdf_abroad_buses_insert,
+                geometry=geopandas.points_from_xy(
+                    gdf_abroad_buses_insert["x"], gdf_abroad_buses_insert["y"]
+                ),
+            )
+            gdf_abroad_buses_insert = gdf_abroad_buses_insert.rename(
+                columns={"geometry": "geom"}
+            ).set_geometry("geom", crs=4326)
+
+            # Insert to db
+            print(gdf_abroad_buses_insert)
+            gdf_abroad_buses_insert.to_postgis(
+                "egon_etrago_bus",
+                engine,
+                schema="grid",
+                index=False,
+                if_exists="append",
+                dtype={"geom": Geometry()},
+            )
+
+            gdf_abroad_buses = pd.concat(
+                [
+                    gdf_abroad_buses,
+                    gdf_abroad_buses_insert
+                ],
+                ignore_index=True,
+            )
+
         return gdf_abroad_buses
 
     else:
@@ -1031,7 +1074,7 @@ class GasNodesAndPipes(Dataset):
     #:
     name: str = "GasNodesAndPipes"
     #:
-    version: str = "0.0.10"
+    version: str = "0.0.11"
 
     tasks = (insert_gas_data_status2019, insert_gas_data)
 

src/egon/data/datasets/gas_neighbours/__init__.py

@@ -13,6 +13,17 @@
     tyndp_gas_generation,
 )
 
+
+def no_gas_neighbours_required():
+    print(
+        """
+          None of the required scenarios need the creation of
+          foreign gas buses
+          """
+    )
+    return
+
+
 tasks = ()
 
 if "eGon2035" in config.settings()["egon-data"]["--scenarios"]:
@@ -26,6 +37,9 @@
 if "eGon100RE" in config.settings()["egon-data"]["--scenarios"]:
     tasks = tasks + (insert_gas_neigbours_eGon100RE,)
 
+if tasks == ():
+    tasks = tasks + (no_gas_neighbours_required,)
+
 
 class GasNeighbours(Dataset):
     def __init__(self, dependencies):

src/egon/data/datasets/hydrogen_etrago/h2_to_ch4.py

@@ -43,6 +43,9 @@ def insert_h2_to_ch4_to_h2():
     target_buses = config.datasets()["etrago_hydrogen"]["targets"]["hydrogen_buses"]
 
 
+    if "status2019" in scenarios:
+        scenarios.remove("status2019")
+
     for scn_name in scenarios:
 
         db.execute_sql(f"""

src/egon/data/datasets/hydrogen_etrago/power_to_h2.py

@@ -43,6 +43,9 @@ def insert_power_to_h2_to_power():
     """
     scenarios = config.settings()["egon-data"]["--scenarios"]
 
+    if "status2019" in scenarios:
+        scenarios.remove("status2019")
+
     for scn_name in scenarios:
 
         # Connect to local database

src/egon/data/datasets/osmtgmod/__init__.py

@@ -583,7 +583,7 @@ def to_pypsa():
               branch_id AS trafo_id,
               f_bus AS bus0,
               t_bus AS bus1,
-              br_x/100 AS x, --- change base from 100MVA (osmtgmod) to 1 MVA (pypsa)
+              br_x/(100 * rate_a) AS x, --- change base from 100MVA (osmtgmod) to the its individual s_nom (pypsa)
               rate_a as s_nom,
               rate_a as s_nom_min,
               TRUE,

src/egon/data/datasets/power_plants/pv_rooftop.py

@@ -160,7 +160,13 @@ def pv_rooftop_per_mv_grid_and_scenario(scenario, level):
             WHERE carrier = 'solar_rooftop'
             AND scenario_name = '{scenario}'
             """
-        ).capacity[0]
+        )
+
+        if target.empty:
+            print(f"No PV rooftop in scenario {scenario}")
+            return
+        else:
+            target = target.capacity[0]
 
         dataset = config.settings()["egon-data"]["--dataset-boundary"]
 

src/egon/data/datasets/power_plants/pv_rooftop_buildings.py

@@ -2583,6 +2583,10 @@ def allocate_scenarios(
     """
     cap_per_bus_id_df = cap_per_bus_id(scenario)
 
+    if cap_per_bus_id_df.empty:
+        print(f"No PV rooftop in scenario {scenario}")
+        return
+
     logger.debug(
         f"cap_per_bus_id_df total capacity: {cap_per_bus_id_df.capacity.sum()}"
     )
@@ -2831,6 +2835,13 @@ def pv_rooftop_to_buildings():
                 <= ts
             ]
 
+        if cap_per_bus_id(scenario).empty:
+            print(f"No PV rooftop in scenario {scenario}")
+            EgonPowerPlantPvRoofBuildingScenario.__table__.create(
+                bind=engine, checkfirst=True
+            )
+            return
+
         logger.debug(f"Desaggregating scenario {scenario}.")
         (
             scenario_buildings_gdf,

src/egon/data/datasets/scenario_parameters/__init__.py

@@ -244,7 +244,7 @@ class ScenarioParameters(Dataset):
     def __init__(self, dependencies):
         super().__init__(
             name="ScenarioParameters",
-            version="0.0.15",
+            version="0.0.17",
             dependencies=dependencies,
             tasks=(
                 create_table,

src/egon/data/datasets/scenario_parameters/parameters.py

@@ -126,7 +126,7 @@ def global_settings(scenario):
         }
     elif scenario == "status2019":
         parameters = {
-            "weather_year": 2019,
+            "weather_year": 2011,
             "population_year": 2019,
             "fuel_costs": {  # TYNDP 2020, data for 2020 (https://2020.entsos-tyndp-scenarios.eu/fuel-commodities-and-carbon-prices/)
                 "oil": 12.9*3.6,  # [EUR/MWh]

src/egon/data/datasets/scenario_path/__init__.py

@@ -0,0 +1,18 @@
+from egon.data.datasets import Dataset
+from egon.data.datasets.scenario_path.import_status2019 import (
+    download_status2019,
+    import_scn_status2019,
+)
+
+
+class CreateIntermediateScenarios(Dataset):
+    def __init__(self, dependencies):
+        super().__init__(
+            name="scenario_path",
+            version="0.0.1",
+            dependencies=dependencies,
+            tasks=(
+                download_status2019,
+                import_scn_status2019,
+            ),
+        )