Merge pull request #345 from openego/features/#342-create-scenations-…

…one-pipeline

Features/#342 create scenations one pipeline

src/egon/data/airflow/dags/pipeline.py

@@ -53,6 +53,7 @@
     HeatSupply,
 )
 from egon.data.datasets.heat_supply.individual_heating import (
+    HeatPumps2019,
     HeatPumps2035,
     HeatPumps2050,
     HeatPumpsPypsaEur,
@@ -589,6 +590,17 @@
         dependencies=[vg250, setup_etrago, create_gas_polygons]
     )
 
+    # Heat pump disaggregation for status2019
+    heat_pumps_2019 = HeatPumps2019(
+        dependencies=[
+            cts_demand_buildings,
+            DistrictHeatingAreas,
+            heat_supply,
+            heat_time_series,
+            power_plants,
+        ]
+    )
+
     # Heat pump disaggregation for eGon2035
     heat_pumps_2035 = HeatPumps2035(
         dependencies=[

src/egon/data/datasets/ch4_prod.py

@@ -198,7 +198,8 @@ def load_biogas_generators(scn_name):
     )
     target_file = Path(".") / "datasets" / "gas_data" / basename
 
-    urlretrieve(url, target_file)
+    if not target_file.is_file():
+        urlretrieve(url, target_file)
 
     # Read-in data from csv-file
     biogas_generators_list = pd.read_excel(

src/egon/data/datasets/chp/__init__.py

@@ -164,6 +164,10 @@ def assign_heat_bus():
             epsg=4326,
         )
 
+        if chp.empty:
+            print(f"No CHP for district heating in scenario {scenario}")
+            return
+
         # Select district heating areas and their centroid
         district_heating = db.select_geodataframe(
             f"""

src/egon/data/datasets/chp_etrago.py

@@ -68,6 +68,10 @@ def insert_egon100re():
         """
     )
 
+    if chp_dh.empty:
+        print("No CHP for district heating in scenario eGon100RE")
+        return
+
     # Create geodataframes for gas CHP plants
     chp_el = link_geom_from_buses(
         gpd.GeoDataFrame(

src/egon/data/datasets/demandregio/__init__.py

@@ -909,10 +909,13 @@ def timeseries_per_wz():
 
     """
     scenarios = egon.data.config.settings()["egon-data"]["--scenarios"]
+    year_already_in_database = []
     for scn in scenarios:
         year = int(scenario_parameters.global_settings(scn)["weather_year"])
         for sector in ["CTS", "industry"]:
-            insert_timeseries_per_wz(sector, int(year))
+            if not year in year_already_in_database:
+                insert_timeseries_per_wz(sector, int(year))
+        year_already_in_database.append(year)
 
 def get_cached_tables():
     """Get cached demandregio tables and db-dump from former runs"""

src/egon/data/datasets/electrical_neighbours.py

@@ -1582,7 +1582,7 @@ def insert_storage_units_sq(scn_name="status2019"):
     None.
 
     """
-    year = int(get_sector_parameters("global", scn_name)["weather_year"])
+    year = int(get_sector_parameters("global", scn_name)["population_year"])
     try:
         sto_sq = entsoe_historic_generation_capacities()
     except:

src/egon/data/datasets/gas_grid.py

@@ -308,6 +308,49 @@ def insert_gas_buses_abroad(scn_name="eGon2035"):
         gdf_abroad_buses.drop_duplicates(
             subset="country", keep="first", inplace=True
         )
+
+        if settings()["egon-data"]["--dataset-boundary"] != "Everything":
+            gdf_abroad_buses_insert = pd.DataFrame(
+                        index=[gdf_abroad_buses.index.max() + 1],
+                        data={
+                            "scn_name": scn_name,
+                            "bus_id": (db.next_etrago_id("bus") + len(gdf_abroad_buses) + 1),
+                            "x": 10.4234469,
+                            "y": 51.0834196,
+                            "country": "DE",
+                            "carrier": gas_carrier,
+                        },
+                    )
+
+            gdf_abroad_buses_insert = geopandas.GeoDataFrame(
+                gdf_abroad_buses_insert,
+                geometry=geopandas.points_from_xy(
+                    gdf_abroad_buses_insert["x"], gdf_abroad_buses_insert["y"]
+                ),
+            )
+            gdf_abroad_buses_insert = gdf_abroad_buses_insert.rename(
+                columns={"geometry": "geom"}
+            ).set_geometry("geom", crs=4326)
+
+            # Insert to db
+            print(gdf_abroad_buses_insert)
+            gdf_abroad_buses_insert.to_postgis(
+                "egon_etrago_bus",
+                engine,
+                schema="grid",
+                index=False,
+                if_exists="append",
+                dtype={"geom": Geometry()},
+            )
+
+            gdf_abroad_buses = pd.concat(
+                [
+                    gdf_abroad_buses,
+                    gdf_abroad_buses_insert
+                ],
+                ignore_index=True,
+            )
+
         return gdf_abroad_buses
 
     else:
@@ -1031,7 +1074,7 @@ class GasNodesAndPipes(Dataset):
     #:
     name: str = "GasNodesAndPipes"
     #:
-    version: str = "0.0.10"
+    version: str = "0.0.11"
 
     tasks = (insert_gas_data_status2019, insert_gas_data)
 

src/egon/data/datasets/hydrogen_etrago/h2_to_ch4.py

@@ -40,6 +40,9 @@ def insert_h2_to_ch4_to_h2():
     """
     scenarios = config.settings()["egon-data"]["--scenarios"]
 
+    if "status2019" in scenarios:
+        scenarios.remove("status2019")
+
     for scn_name in scenarios:
         # Connect to local database
         engine = db.engine()

src/egon/data/datasets/hydrogen_etrago/power_to_h2.py

@@ -43,6 +43,9 @@ def insert_power_to_h2_to_power():
     """
     scenarios = config.settings()["egon-data"]["--scenarios"]
 
+    if "status2019" in scenarios:
+        scenarios.remove("status2019")
+
     for scn_name in scenarios:
 
         # Connect to local database

src/egon/data/datasets/power_plants/pv_rooftop.py

@@ -160,7 +160,13 @@ def pv_rooftop_per_mv_grid_and_scenario(scenario, level):
             WHERE carrier = 'solar_rooftop'
             AND scenario_name = '{scenario}'
             """
-        ).capacity[0]
+        )
+
+        if target.empty:
+            print(f"No PV rooftop in scenario {scenario}")
+            return
+        else:
+            target = target.capacity[0]
 
         dataset = config.settings()["egon-data"]["--dataset-boundary"]
 

src/egon/data/datasets/power_plants/pv_rooftop_buildings.py

@@ -2583,6 +2583,10 @@ def allocate_scenarios(
     """
     cap_per_bus_id_df = cap_per_bus_id(scenario)
 
+    if cap_per_bus_id_df.empty:
+        print(f"No PV rooftop in scenario {scenario}")
+        return
+
     logger.debug(
         f"cap_per_bus_id_df total capacity: {cap_per_bus_id_df.capacity.sum()}"
     )
@@ -2831,6 +2835,13 @@ def pv_rooftop_to_buildings():
                 <= ts
             ]
 
+        if cap_per_bus_id(scenario).empty:
+            print(f"No PV rooftop in scenario {scenario}")
+            EgonPowerPlantPvRoofBuildingScenario.__table__.create(
+                bind=engine, checkfirst=True
+            )
+            return
+
         logger.debug(f"Desaggregating scenario {scenario}.")
         (
             scenario_buildings_gdf,

src/egon/data/datasets/scenario_parameters/__init__.py

@@ -244,7 +244,7 @@ class ScenarioParameters(Dataset):
     def __init__(self, dependencies):
         super().__init__(
             name="ScenarioParameters",
-            version="0.0.15",
+            version="0.0.17",
             dependencies=dependencies,
             tasks=(
                 create_table,

src/egon/data/datasets/scenario_parameters/parameters.py

@@ -126,7 +126,7 @@ def global_settings(scenario):
         }
     elif scenario == "status2019":
         parameters = {
-            "weather_year": 2019,
+            "weather_year": 2011,
             "population_year": 2019,
             "fuel_costs": {  # TYNDP 2020, data for 2020 (https://2020.entsos-tyndp-scenarios.eu/fuel-commodities-and-carbon-prices/)
                 "oil": 12.9*3.6,  # [EUR/MWh]

src/egon/data/datasets/scenario_path/import_status2019.py

@@ -24,6 +24,12 @@ def download_status2019():
     None.
 
     """
+
+    if "status2019" in config.settings()["egon-data"]["--scenarios"]:
+        print("Scenario status2019 was created within the pipeline. "
+              "It is not imported from zenodo.")
+        return
+
     # Get parameters from config and set download URL
     url = sources["url_status2019"]
     status2019_path = Path(".") / "PoWerD_status2019.backup"
@@ -43,6 +49,12 @@ def import_scn_status2019():
     *No parameters required
 
     """
+
+    if "status2019" in config.settings()["egon-data"]["--scenarios"]:
+        print("Scenario status2019 was created within the pipeline. "
+              "It is not imported from zenodo.")
+        return
+
     # Connect to the data base
     con = db.engine()