diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 160ddd430..34d88c305 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -25,12 +25,9 @@ jobs: strategy: matrix: python-version: - - "3.7" - "3.8" platform: [ubuntu-latest, macos-latest] exclude: - - platform: macos-latest - python-version: "3.7" - platform: macos-latest python-version: "3.8" max-parallel: 4 diff --git a/AUTHORS.rst b/AUTHORS.rst index 11f59b067..4ad94b63c 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -2,4 +2,4 @@ Authors ======= -* Guido Pleßmann, Ilka Cußmann, Stephan Günther, Jonathan Amme, Julian Endres, Kilian Helfenbein - https://github.com/openego/eGon-data +* Jonathan Amme, Clara Büttner, Ilka Cußman, Julian Endres, Carlos Epia, Kilian Helfenbein, Stephan Günther, Ulf Müller, Amélia Nadal, Guido Pleßmann, Francesco Witte - https://github.com/openego/eGon-data diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b65013c00..f30a42a04 100755 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -252,6 +252,16 @@ Added `PR #903 `_ * Add low flex scenario 'eGon2035_lowflex' `#822 `_ +* Add Status2023 update heat demand + `#199 `_ +* Add status2023 electrical neighbours + `#182 `_ +* Add Status2023 to gas nodes + `#205 `_ +* Update osm for status2023 + `#169 `_ +* Add wrapped_partial to dynamise task generation + `#207 `_ * Add MaStR geocoding and handling of conventional generators `#1095 `_ @@ -494,12 +504,14 @@ Changed `#1073 `_ * Change hgv data source to use database `#1086 `_ +* Rename eMob MIT carrier names (use underscores) + `#1105 `_ * Change desposit ID for data_bundle download from zenodo sandbox `#1110 `_ * Use MaStR geocoding results for pv rooftop to buildings mapping workflow `#1095 `_ -* Rename eMob MIT carrier names (use underscores) - `#1105 `_ +* Add eMob MIT for SQ2023 scenario + `#176 `_ .. _#799: https://github.com/openego/eGon-data/issues/799 diff --git a/README.rst b/README.rst index e2e4a2509..7c5e4aa77 100644 --- a/README.rst +++ b/README.rst @@ -137,6 +137,18 @@ packages are required too. Right now these are: You also have to agree on the `terms of use `_ +* To download generation and demand data from entsoe you need to register in + the `entsoe platform `_. Once the user is + created, send an email to transparency@entsoe.eu with “Restful API access” in + the subject line. Indicate the email address you entered during registration + in the email body. After authorisation a personal token can be generated in + 'My Account Settings'to access the available data. This token must be saved + in a text file called *.entsoe-token* in the home directory. + In Ubuntu you can go to the home directory by typing :code:`cd ~` in a + terminal. To create the file execute :code:`nano ".entsoe-token"` and then + paste your personal token (36 characters). Finally press :code:`CRL + x` to + save and exit. + * Make sure you have enough free disk space (~350 GB) in your working directory. diff --git a/setup.py b/setup.py index c073ae376..f7b870fc2 100755 --- a/setup.py +++ b/setup.py @@ -59,7 +59,6 @@ def read(*names, **kwargs): "Operating System :: Microsoft :: Windows", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: Implementation :: CPython", # uncomment if you test on these interpreters: @@ -79,39 +78,45 @@ def read(*names, **kwargs): keywords=[ # eg: 'keyword1', 'keyword2', 'keyword3', ], - python_requires=">=3.7", + python_requires=">=3.8", install_requires=[ # eg: 'aspectlib==1.1.1', 'six>=1.7', - "apache-airflow>=1.10.14,<2.0", # See accompanying commit message - "atlite==0.2.5", + "apache-airflow>2.0", + "apache-airflow[postgres]", + "apache-airflow-providers-sendgrid", + "atlite==0.2.11", "cdsapi", - "click", - "geopandas>=0.10.0,<0.11.0", + "click<8.1", + "entsoe-py >=0.6.2", + "Flask-Session<0.6.0", + "GeoAlchemy2", + "geopandas>=0.10.0", "geopy", - "geovoronoi==0.3.0", - "importlib-resources", + "geovoronoi", + "importlib-resources<6.0", "loguru", - "markupsafe<2.1.0", # MarkupSafe>=2.1.0 breaks WTForms<3 + "markupsafe", "matplotlib", "netcdf4", - "numpy<1.23", # incompatibilities with shapely 1.7. - # See: https://stackoverflow.com/a/73354885/12460232 - "oedialect==0.0.8", + "numpy", "omi", "openpyxl", - "pandas>1.2.0,<1.4", # pandas>=1.4 needs SQLAlchemy>=1.4 + "pandas>2.0.0", "psycopg2", "pyaml", - "pypsa==0.17.1", + "pypsa==0.20.1", + "pydantic<2.0", "rasterio", + "requests", "rioxarray", "rtree", "saio", "seaborn", + "setuptools>60.0", "shapely", "snakemake<7", - "sqlalchemy<1.4", # Airflow<2.0 is not compatible with SQLAlchemy>=1.4 - "wtforms<3", # WTForms>=3.0 breaks Airflow<2.0 + "sqlalchemy", + "wtforms", "xarray", "xlrd", ], diff --git a/src/egon/data/__init__.py b/src/egon/data/__init__.py index 22de0bf2b..f0b261789 100644 --- a/src/egon/data/__init__.py +++ b/src/egon/data/__init__.py @@ -2,14 +2,14 @@ from loguru import logger import click - +import shutil __version__ = "0.0.0" def echo(message): prefix, message = message.split(" - ") lines = message.split("\n") - width = min(72, click.get_terminal_size()[0]) + width = min(72, shutil.get_terminal_size()[0]) wraps = ["\n".join(wrap(line, width)) for line in lines] message = "\n".join([prefix] + wraps) click.echo(message, err=True) diff --git a/src/egon/data/airflow/airflow.cfg b/src/egon/data/airflow/airflow.cfg index 80b7247f7..faf9fba9e 100755 --- a/src/egon/data/airflow/airflow.cfg +++ b/src/egon/data/airflow/airflow.cfg @@ -7,48 +7,6 @@ dags_folder = {dags} # This path must be absolute # base_log_folder = ${{AIRFLOW_HOME}}/logs -# Airflow can store logs remotely in AWS S3, Google Cloud Storage or Elastic Search. -# Set this to True if you want to enable remote logging. -remote_logging = False - -# Users must supply an Airflow connection id that provides access to the storage -# location. -remote_log_conn_id = -remote_base_log_folder = -encrypt_s3_logs = False - -# Logging level -logging_level = INFO - -# Logging level for Flask-appbuilder UI -fab_logging_level = WARN - -# Logging class -# Specify the class that will specify the logging configuration -# This class has to be on the python classpath -# Example: logging_config_class = my.path.default_local_settings.LOGGING_CONFIG -logging_config_class = - -# Flag to enable/disable Colored logs in Console -# Colour the logs when the controlling terminal is a TTY. -colored_console_log = True - -# Log format for when Colored logs is enabled -colored_log_format = [%%(blue)s%%(asctime)s%%(reset)s] {{%%(blue)s%%(filename)s:%%(reset)s%%(lineno)d}} %%(log_color)s%%(levelname)s%%(reset)s - %%(log_color)s%%(message)s%%(reset)s -colored_formatter_class = airflow.utils.log.colored_log.CustomTTYColoredFormatter - -# Format of Log line -log_format = [%%(asctime)s] {{%%(filename)s:%%(lineno)d}} %%(levelname)s - %%(message)s -simple_log_format = %%(asctime)s %%(levelname)s - %%(message)s - -# Log filename format -log_filename_template = {{{{ ti.dag_id }}}}/{{{{ ti.task_id }}}}/{{{{ ts }}}}/{{{{ try_number }}}}.log -log_processor_filename_template = {{{{ filename }}}}.log -# dag_processor_manager_log_location = ${{AIRFLOW_HOME}}/logs/dag_processor_manager/dag_processor_manager.log - -# Name of handler to read task instance logs. -# Default to use task handler. -task_log_reader = task # Hostname by providing a path to a callable, which will resolve the hostname. # The format is "package:function". @@ -58,7 +16,7 @@ task_log_reader = task # # No argument should be required in the function specified. # If using IP address as hostname is preferred, use value ``airflow.utils.net:get_host_ip_address`` -hostname_callable = socket:getfqdn +hostname_callable = socket.getfqdn # Default timezone in case supplied date times are naive # can be utc (default), system, or any IANA timezone string (e.g. Europe/Amsterdam) @@ -68,53 +26,8 @@ default_timezone = utc # SequentialExecutor, LocalExecutor, CeleryExecutor, DaskExecutor, KubernetesExecutor executor = LocalExecutor -# The SqlAlchemy connection string to the metadata database. -# SqlAlchemy supports many different database engine, more information -# their website -sql_alchemy_conn = postgresql+psycopg2://{--database-user}:{--database-password}@{--database-host}:{--database-port}/{--airflow-database-name} - -# The encoding for the databases -sql_engine_encoding = utf-8 -# If SqlAlchemy should pool database connections. -sql_alchemy_pool_enabled = True -# The SqlAlchemy pool size is the maximum number of database connections -# in the pool. 0 indicates no limit. -sql_alchemy_pool_size = 5 - -# The maximum overflow size of the pool. -# When the number of checked-out connections reaches the size set in pool_size, -# additional connections will be returned up to this limit. -# When those additional connections are returned to the pool, they are disconnected and discarded. -# It follows then that the total number of simultaneous connections the pool will allow -# is pool_size + max_overflow, -# and the total number of "sleeping" connections the pool will allow is pool_size. -# max_overflow can be set to -1 to indicate no overflow limit; -# no limit will be placed on the total number of concurrent connections. Defaults to 10. -sql_alchemy_max_overflow = 10 - -# The SqlAlchemy pool recycle is the number of seconds a connection -# can be idle in the pool before it is invalidated. This config does -# not apply to sqlite. If the number of DB connections is ever exceeded, -# a lower config value will allow the system to recover faster. -sql_alchemy_pool_recycle = 1800 - -# Check connection at the start of each connection pool checkout. -# Typically, this is a simple statement like "SELECT 1". -# More information here: -# https://docs.sqlalchemy.org/en/13/core/pooling.html#disconnect-handling-pessimistic -sql_alchemy_pool_pre_ping = True - -# The schema to use for the metadata database. -# SqlAlchemy supports databases with the concept of multiple schemas. -sql_alchemy_schema = - -# Import path for connect args in SqlAlchemy. Default to an empty dict. -# This is useful when you want to configure db engine args that SqlAlchemy won't parse -# in connection string. -# See https://docs.sqlalchemy.org/en/13/core/engines.html#sqlalchemy.create_engine.params.connect_args -# sql_alchemy_connect_args = # The amount of parallelism as a setting to the executor. This defines # the max number of task instances that should run simultaneously @@ -122,7 +35,7 @@ sql_alchemy_schema = parallelism = 32 # The number of task instances allowed to run concurrently by the scheduler -dag_concurrency = {--jobs} +max_active_tasks_per_dag = {--jobs} # Are DAGs paused by default at creation dags_are_paused_at_creation = False @@ -229,6 +142,100 @@ check_slas = True # Example: xcom_backend = path.to.CustomXCom xcom_backend = airflow.models.xcom.BaseXCom +# UI to hide sensitive variable fields when set to True +hide_sensitive_variable_fields = True + +[database] +# The SqlAlchemy connection string to the metadata database. +# SqlAlchemy supports many different database engine, more information +# their website +sql_alchemy_conn = postgresql+psycopg2://{--database-user}:{--database-password}@{--database-host}:{--database-port}/{--airflow-database-name} + +# The encoding for the databases +sql_engine_encoding = utf-8 + +# If SqlAlchemy should pool database connections. +sql_alchemy_pool_enabled = True + +# The SqlAlchemy pool size is the maximum number of database connections +# in the pool. 0 indicates no limit. +sql_alchemy_pool_size = 5 + +# The maximum overflow size of the pool. +# When the number of checked-out connections reaches the size set in pool_size, +# additional connections will be returned up to this limit. +# When those additional connections are returned to the pool, they are disconnected and discarded. +# It follows then that the total number of simultaneous connections the pool will allow +# is pool_size + max_overflow, +# and the total number of "sleeping" connections the pool will allow is pool_size. +# max_overflow can be set to -1 to indicate no overflow limit; +# no limit will be placed on the total number of concurrent connections. Defaults to 10. +sql_alchemy_max_overflow = 10 + +# The SqlAlchemy pool recycle is the number of seconds a connection +# can be idle in the pool before it is invalidated. This config does +# not apply to sqlite. If the number of DB connections is ever exceeded, +# a lower config value will allow the system to recover faster. +sql_alchemy_pool_recycle = 1800 + +# Check connection at the start of each connection pool checkout. +# Typically, this is a simple statement like "SELECT 1". +# More information here: +# https://docs.sqlalchemy.org/en/13/core/pooling.html#disconnect-handling-pessimistic +sql_alchemy_pool_pre_ping = True + +# The schema to use for the metadata database. +# SqlAlchemy supports databases with the concept of multiple schemas. +sql_alchemy_schema = + +# Import path for connect args in SqlAlchemy. Default to an empty dict. +# This is useful when you want to configure db engine args that SqlAlchemy won't parse +# in connection string. +# See https://docs.sqlalchemy.org/en/13/core/engines.html#sqlalchemy.create_engine.params.connect_args +# sql_alchemy_connect_args = + +[logging] +# Logging level +logging_level = INFO + +# Logging level for Flask-appbuilder UI +fab_logging_level = WARN + +# Logging class +# Specify the class that will specify the logging configuration +# This class has to be on the python classpath +# Example: logging_config_class = my.path.default_local_settings.LOGGING_CONFIG +logging_config_class = + +# Flag to enable/disable Colored logs in Console +# Colour the logs when the controlling terminal is a TTY. +colored_console_log = False + +# Log format for when Colored logs is enabled +#colored_log_format = [%%(blue)s%%(asctime)s%%(reset)s] {{%%(blue)s%%(filename)s:%%(reset)s%%(lineno)d}} %%(log_color)s%%(levelname)s%%(reset)s - %%(log_color)s%%(message)s%%(reset)s +#colored_formatter_class = airflow.utils.log.colored_log.CustomTTYColoredFormatter + + +# Format of Log line +log_format = [%%(asctime)s] {{%%(filename)s:%%(lineno)d}} %%(levelname)s - %%(message)s +simple_log_format = %%(asctime)s %%(levelname)s - %%(message)s + +# Log filename format +log_filename_template = {{{{ ti.dag_id }}}}/{{{{ ti.task_id }}}}/{{{{ ts }}}}/{{{{ try_number }}}}.log + +log_processor_filename_template = {{{{ filename }}}}.log + + +# Airflow can store logs remotely in AWS S3, Google Cloud Storage or Elastic Search. +# Set this to True if you want to enable remote logging. +remote_logging = False + +# Users must supply an Airflow connection id that provides access to the storage +# location. +remote_log_conn_id = +remote_base_log_folder = +encrypt_s3_logs = False + [secrets] # Full class name of secrets backend to enable (will precede env vars and metastore in search path) # Example: backend = airflow.contrib.secrets.aws_systems_manager.SystemsManagerParameterStoreBackend @@ -260,7 +267,7 @@ fail_fast = False # How to authenticate users of the API. See # https://airflow.apache.org/docs/stable/security.html for possible values. # ("airflow.api.auth.backend.default" allows all requests for historic reasons) -auth_backend = airflow.api.auth.backend.deny_all +auth_backends = airflow.api.auth.backend.session [lineage] # what lineage backend to use @@ -276,12 +283,17 @@ password = [operators] # The default owner assigned to each new operator, unless # provided explicitly or passed via ``default_args`` -default_owner = airflow +default_owner = airflowstatsd_on = False default_cpus = 1 default_ram = 512 default_disk = 512 default_gpus = 0 + + +# Default queue that tasks get assigned to and that worker listen on. +default_queue = default + [hive] # Default mapreduce queue for HiveOperator tasks default_hive_mapred_queue = @@ -372,7 +384,7 @@ owner_mode = user # Default DAG view. Valid values are: # tree, graph, duration, gantt, landing_times -dag_default_view = tree +dag_default_view = graph # "Default DAG orientation. Valid values are:" # LR (Left->Right), TB (Top->Bottom), RL (Right->Left), BT (Bottom->Top) @@ -406,7 +418,7 @@ page_size = 100 rbac = False # Define the color of navigation bar -navbar_color = #007A87 +navbar_color = #fff # Default dagrun to show in UI default_dag_run_display_number = 25 @@ -458,22 +470,22 @@ update_fab_perms = True # session_lifetime_minutes = 43200 [email] -email_backend = airflow.utils.email.send_email_smtp +email_backend = airflow.providers.sendgrid.utils.emailer.send_email +email_conn_id = sendgrid_default +from_email = "powerddata@gmail.com" [smtp] # If you want airflow to send emails on retries, failure, and you want to use # the airflow.utils.email.send_email_smtp function, you have to configure an # smtp server here -smtp_host = localhost -smtp_starttls = True +smtp_host = smtp.sendgrid.net +smtp_starttls = False smtp_ssl = False -# Example: smtp_user = airflow -# smtp_user = -# Example: smtp_password = airflow -# smtp_password = -smtp_port = 25 -smtp_mail_from = airflow@example.com +smtp_user = apikey +smtp_password = +smtp_port = 587 +smtp_mail_from = powerddata@gmail.com [sentry] @@ -539,8 +551,6 @@ flower_port = 5555 # Example: flower_basic_auth = user1:password1,user2:password2 flower_basic_auth = -# Default queue that tasks get assigned to and that worker listen on. -default_queue = default # How many processes CeleryExecutor uses to sync task state. # 0 means to use max(1, number of cores - 1) processes. @@ -654,7 +664,7 @@ catchup_by_default = True max_tis_per_query = 512 # Statsd (https://github.com/etsy/statsd) integration settings -statsd_on = False + statsd_host = localhost statsd_port = 8125 statsd_prefix = airflow @@ -677,6 +687,9 @@ use_job_schedule = True # Only has effect if schedule_interval is set to None in DAG allow_trigger_in_future = False +[metrics] +statsd_on = False + [ldap] # set this to ldaps://: uri = @@ -754,8 +767,7 @@ keytab = airflow.keytab api_rev = v3 [admin] -# UI to hide sensitive variable fields when set to True -hide_sensitive_variable_fields = True + [elasticsearch] # Elasticsearch host @@ -785,6 +797,10 @@ json_fields = asctime, filename, lineno, levelname, message use_ssl = False verify_certs = True +[kubernetes_executor] +# The Kubernetes namespace where airflow workers should be created. Defaults to ``default`` +namespace = default + [kubernetes] # The repository, tag and imagePullPolicy of the Kubernetes Image for the Worker to Run worker_container_repository = @@ -805,8 +821,7 @@ delete_worker_pods_on_failure = False # Number of Kubernetes Worker Pod creation calls per scheduler loop worker_pods_creation_batch_size = 1 -# The Kubernetes namespace where airflow workers should be created. Defaults to ``default`` -namespace = default + # Allows users to launch pods in multiple namespaces. # Will require creating a cluster-role for the scheduler diff --git a/src/egon/data/airflow/dags/pipeline.py b/src/egon/data/airflow/dags/pipeline.py index 4156ab353..0f438b548 100755 --- a/src/egon/data/airflow/dags/pipeline.py +++ b/src/egon/data/airflow/dags/pipeline.py @@ -4,6 +4,7 @@ import airflow from egon.data.config import set_numexpr_threads +from egon.data.metadata import Json_Metadata from egon.data.datasets import database from egon.data.datasets.calculate_dlr import Calculate_dlr from egon.data.datasets.ch4_prod import CH4Production @@ -95,7 +96,6 @@ from egon.data.datasets.zensus import ZensusMiscellaneous, ZensusPopulation from egon.data.datasets.zensus_mv_grid_districts import ZensusMvGridDistricts from egon.data.datasets.zensus_vg250 import ZensusVg250 - # Set number of threads used by numpy and pandas set_numexpr_threads() @@ -113,7 +113,6 @@ is_paused_upon_creation=False, schedule_interval=None, ) as pipeline: - tasks = pipeline.task_dict setup = database.Setup() @@ -581,7 +580,7 @@ heat_supply, heat_time_series, heat_pumps_pypsa_eur_sec, - tasks["power_plants.pv_rooftop_buildings.pv-rooftop-to-buildings"], + power_plants, ] ) @@ -630,24 +629,6 @@ ] ) - mit_charging_infrastructure = MITChargingInfrastructure( - dependencies=[mv_grid_districts, hh_demand_buildings_setup] - ) - - # eMobility: heavy duty transport - heavy_duty_transport = HeavyDutyTransport( - dependencies=[vg250, setup_etrago, create_gas_polygons_egon2035] - ) - - cts_demand_buildings = CtsDemandBuildings( - dependencies=[ - osm_buildings_streets, - cts_electricity_demand_annual, - hh_demand_buildings_setup, - tasks["heat_demand_timeseries.export-etrago-cts-heat-profiles"], - ] - ) - # Create load areas load_areas = LoadArea( dependencies=[ @@ -689,3 +670,12 @@ low_flex_scenario, ] ) + + # upload json metadata at the end + json_metadata = Json_Metadata( + dependencies=[ + load_areas, + cts_demand_buildings, + heat_pumps_2050 + ] + ) diff --git a/src/egon/data/airflow/dags/pipeline_status_quo.py b/src/egon/data/airflow/dags/pipeline_status_quo.py new file mode 100755 index 000000000..4400248c9 --- /dev/null +++ b/src/egon/data/airflow/dags/pipeline_status_quo.py @@ -0,0 +1,524 @@ +import os + +from airflow.utils.dates import days_ago +import airflow + +from egon.data.config import set_numexpr_threads +from egon.data.datasets import database +from egon.data.datasets.ch4_prod import CH4Production +from egon.data.datasets.ch4_storages import CH4Storages +from egon.data.datasets.chp import Chp +from egon.data.datasets.chp_etrago import ChpEtrago +from egon.data.datasets.data_bundle import DataBundle +from egon.data.datasets.demandregio import DemandRegio +from egon.data.datasets.district_heating_areas import DistrictHeatingAreas +from egon.data.datasets.electrical_neighbours import ElectricalNeighbours +from egon.data.datasets.electricity_demand import ( + CtsElectricityDemand, + HouseholdElectricityDemand, +) +from egon.data.datasets.electricity_demand_etrago import ElectricalLoadEtrago +from egon.data.datasets.electricity_demand_timeseries import ( + hh_buildings, + hh_profiles, +) +from egon.data.datasets.electricity_demand_timeseries.cts_buildings import ( + CtsDemandBuildings, +) +from egon.data.datasets.emobility.motorized_individual_travel import ( + MotorizedIndividualTravel, +) +from egon.data.datasets.emobility.motorized_individual_travel_charging_infrastructure import ( # noqa: E501 + MITChargingInfrastructure, +) +from egon.data.datasets.era5 import WeatherData +from egon.data.datasets.etrago_setup import EtragoSetup +from egon.data.datasets.fill_etrago_gen import Egon_etrago_gen +from egon.data.datasets.fix_ehv_subnetworks import FixEhvSubnetworks +from egon.data.datasets.gas_areas import GasAreasStatusQuo +from egon.data.datasets.gas_grid import GasNodesAndPipes +from egon.data.datasets.gas_neighbours import GasNeighbours +from egon.data.datasets.heat_demand import HeatDemandImport +from egon.data.datasets.heat_demand_europe import HeatDemandEurope +from egon.data.datasets.heat_demand_timeseries import HeatTimeSeries +from egon.data.datasets.heat_etrago import HeatEtrago +from egon.data.datasets.heat_etrago.hts_etrago import HtsEtragoTable +from egon.data.datasets.heat_supply import HeatSupply +from egon.data.datasets.heat_supply.individual_heating import HeatPumpsStatusQuo +from egon.data.datasets.industrial_sites import MergeIndustrialSites +from egon.data.datasets.industry import IndustrialDemandCurves +from egon.data.datasets.loadarea import LoadArea, OsmLanduse +from egon.data.datasets.mastr import mastr_data_setup +from egon.data.datasets.mv_grid_districts import mv_grid_districts_setup +from egon.data.datasets.osm import OpenStreetMap +from egon.data.datasets.osm_buildings_streets import OsmBuildingsStreets +from egon.data.datasets.osmtgmod import Osmtgmod +from egon.data.datasets.power_etrago import OpenCycleGasTurbineEtrago +from egon.data.datasets.power_plants import PowerPlants +from egon.data.datasets.pypsaeursec import PypsaEurSec +from egon.data.datasets.renewable_feedin import RenewableFeedin +from egon.data.datasets.scenario_capacities import ScenarioCapacities +from egon.data.datasets.scenario_parameters import ScenarioParameters +from egon.data.datasets.society_prognosis import SocietyPrognosis +from egon.data.datasets.storages import Storages +from egon.data.datasets.storages_etrago import StorageEtrago +from egon.data.datasets.substation import SubstationExtraction +from egon.data.datasets.substation_voronoi import SubstationVoronoi +from egon.data.datasets.tyndp import Tyndp +from egon.data.datasets.vg250 import Vg250 +from egon.data.datasets.vg250_mv_grid_districts import Vg250MvGridDistricts +from egon.data.datasets.zensus import ZensusMiscellaneous, ZensusPopulation +from egon.data.datasets.zensus_mv_grid_districts import ZensusMvGridDistricts +from egon.data.datasets.zensus_vg250 import ZensusVg250 + +# Set number of threads used by numpy and pandas +set_numexpr_threads() + + + +with airflow.DAG( + "powerd-status-quo-processing-pipeline", + description="The PoWerD Status Quo data processing DAG.", + default_args={"start_date": days_ago(1), + "email_on_failure": False, + "email":"clara.buettner@hs-flensburg.de"}, + template_searchpath=[ + os.path.abspath( + os.path.join( + os.path.dirname(__file__), "..", "..", "processing", "vg250" + ) + ) + ], + is_paused_upon_creation=False, + schedule_interval=None, +) as pipeline: + + tasks = pipeline.task_dict + + setup = database.Setup() + + osm = OpenStreetMap(dependencies=[setup]) + + data_bundle = DataBundle(dependencies=[setup]) + + # Import VG250 (Verwaltungsgebiete 250) data + vg250 = Vg250(dependencies=[setup]) + + # Scenario table + scenario_parameters = ScenarioParameters(dependencies=[setup]) + + # Download TYNDP data + tyndp_data = Tyndp(dependencies=[setup]) #TODO: kick out or adjust + + # Import zensus population + zensus_population = ZensusPopulation(dependencies=[setup, vg250]) + + # Combine zensus and VG250 data + zensus_vg250 = ZensusVg250(dependencies=[vg250, zensus_population]) + + # Download and import zensus data on households, buildings and apartments + zensus_miscellaneous = ZensusMiscellaneous( + dependencies=[zensus_population, zensus_vg250] + ) + + # Import DemandRegio data + demandregio = DemandRegio( + dependencies=[data_bundle, scenario_parameters, setup, vg250] + ) + + # Society prognosis + society_prognosis = SocietyPrognosis( + dependencies=[demandregio, zensus_miscellaneous] + ) + + # OSM (OpenStreetMap) buildings, streets and amenities + osm_buildings_streets = OsmBuildingsStreets( + dependencies=[osm, zensus_miscellaneous] + ) + + # Import weather data + weather_data = WeatherData( + dependencies=[scenario_parameters, setup, vg250] + ) + + # Future national heat demands for foreign countries based on Hotmaps + # download only, processing in PyPSA-Eur-Sec fork + hd_abroad = HeatDemandEurope(dependencies=[setup]) + + # Set eTraGo input tables + setup_etrago = EtragoSetup(dependencies=[setup]) + + substation_extraction = SubstationExtraction(dependencies=[osm, vg250]) + + # Generate the osmTGmod ehv/hv grid model + osmtgmod = Osmtgmod( + dependencies=[ + scenario_parameters, + setup_etrago, + substation_extraction, + tasks["osm.download"], + ] + ) + + # Fix eHV subnetworks in Germany manually + fix_subnetworks = FixEhvSubnetworks(dependencies=[osmtgmod]) + + # Retrieve MaStR (Marktstammdatenregister) data + mastr_data = mastr_data_setup(dependencies=[setup]) + + # Create Voronoi polygons + substation_voronoi = SubstationVoronoi( + dependencies=[tasks["osmtgmod.substation.extract"], vg250] + ) + + # MV (medium voltage) grid districts + mv_grid_districts = mv_grid_districts_setup( + dependencies=[substation_voronoi] + ) + + # Calculate future heat demand based on Peta5_0_1 data + heat_demand_Germany = HeatDemandImport( + dependencies=[scenario_parameters, vg250, zensus_vg250] + ) + + # Extract landuse areas from the `osm` dataset + osm_landuse = OsmLanduse(dependencies=[osm, vg250]) + + # Calculate feedin from renewables + renewable_feedin = RenewableFeedin( + dependencies=[vg250, zensus_vg250, weather_data] + ) + + # Demarcate district heating areas + district_heating_areas = DistrictHeatingAreas( + dependencies=[ + heat_demand_Germany, + scenario_parameters, + zensus_miscellaneous, + ] + ) + + # TODO: What does "trans" stand for? + # Calculate dynamic line rating for HV (high voltage) trans lines + # dlr = Calculate_dlr( + # dependencies=[data_bundle, osmtgmod, weather_data] # , fix_subnetworks] + #) + + # Map zensus grid districts + zensus_mv_grid_districts = ZensusMvGridDistricts( + dependencies=[mv_grid_districts, zensus_population] + ) + + # Map federal states to mv_grid_districts + vg250_mv_grid_districts = Vg250MvGridDistricts( + dependencies=[mv_grid_districts, vg250] + ) + + # Create household demand profiles on zensus level + hh_demand_profiles_setup = hh_profiles.HouseholdDemands( + dependencies=[ + demandregio, + tasks[ + "osm_buildings_streets" + ".create-buildings-residential-zensus-mapping" + ], + vg250, + zensus_miscellaneous, + zensus_mv_grid_districts, + zensus_vg250, + ] + ) + + # Household electricity demand buildings + hh_demand_buildings_setup = hh_buildings.setup( + dependencies=[ + tasks[ + "electricity_demand_timeseries" + ".hh_profiles" + ".houseprofiles-in-census-cells" + ] + ] + ) + + # Get household electrical demands for cencus cells + household_electricity_demand_annual = HouseholdElectricityDemand( + dependencies=[ + tasks[ + "electricity_demand_timeseries" + ".hh_buildings" + ".map-houseprofiles-to-buildings" + ] + ] + ) + + # Distribute electrical CTS demands to zensus grid + cts_electricity_demand_annual = CtsElectricityDemand( + dependencies=[ + demandregio, + heat_demand_Germany, + # household_electricity_demand_annual, + tasks["electricity_demand.create-tables"], + tasks["etrago_setup.create-tables"], + zensus_mv_grid_districts, + zensus_vg250, + ] + ) + + # Industry + industrial_sites = MergeIndustrialSites( + dependencies=[data_bundle, setup, vg250] + ) + demand_curves_industry = IndustrialDemandCurves( + dependencies=[ + demandregio, + industrial_sites, + osm_landuse, + mv_grid_districts, + osm, + ] + ) + + # Electrical loads to eTraGo + electrical_load_etrago = ElectricalLoadEtrago( + dependencies=[ + cts_electricity_demand_annual, + demand_curves_industry, + hh_demand_buildings_setup, + household_electricity_demand_annual, + ] + ) + + # Heat time Series + heat_time_series = HeatTimeSeries( + dependencies=[ + data_bundle, + demandregio, + heat_demand_Germany, + district_heating_areas, + vg250, + zensus_mv_grid_districts, + hh_demand_buildings_setup, + weather_data, + ] + ) + + cts_demand_buildings = CtsDemandBuildings( + dependencies=[ + osm_buildings_streets, + cts_electricity_demand_annual, + hh_demand_buildings_setup, + tasks["heat_demand_timeseries.export-etrago-cts-heat-profiles"], + ] + ) + + # run pypsa-eur-sec + run_pypsaeursec = PypsaEurSec( + dependencies=[ + weather_data, + hd_abroad, + osmtgmod, + setup_etrago, + data_bundle, + electrical_load_etrago, + heat_time_series, + ] + ) + + # Deal with electrical neighbours + foreign_lines = ElectricalNeighbours( + dependencies=[run_pypsaeursec, tyndp_data] + ) + + # Import NEP (Netzentwicklungsplan) data + scenario_capacities = ScenarioCapacities( + dependencies=[ + data_bundle, + run_pypsaeursec, + setup, + vg250, + zensus_population, + ] + ) + + # Import gas grid + gas_grid_insert_data = GasNodesAndPipes( + dependencies=[ + data_bundle, + foreign_lines, + osmtgmod, + scenario_parameters, + tasks["etrago_setup.create-tables"], + ] + ) + # Create gas voronoi status quo + create_gas_polygons_statusquo = GasAreasStatusQuo( + dependencies=[setup_etrago, vg250, gas_grid_insert_data, substation_voronoi] + ) + + # Gas abroad + gas_abroad_insert_data = GasNeighbours( + dependencies=[ + gas_grid_insert_data, + run_pypsaeursec, + foreign_lines, + create_gas_polygons_statusquo, + ] + ) + + # Import gas production + gas_production_insert_data = CH4Production( + dependencies=[create_gas_polygons_statusquo] + ) + + # Import CH4 storages + insert_data_ch4_storages = CH4Storages( + dependencies=[create_gas_polygons_statusquo] + ) + + # CHP locations + chp = Chp( + dependencies=[ + create_gas_polygons_statusquo, + demand_curves_industry, + district_heating_areas, + industrial_sites, + osm_landuse, + mastr_data, + mv_grid_districts, + ] + ) + + # Power plants + power_plants = PowerPlants( + dependencies=[ + chp, + cts_electricity_demand_annual, + household_electricity_demand_annual, + mastr_data, + mv_grid_districts, + renewable_feedin, + scenario_parameters, + setup, + substation_extraction, + tasks["etrago_setup.create-tables"], + vg250_mv_grid_districts, + zensus_mv_grid_districts, + ] + ) + + create_ocgt = OpenCycleGasTurbineEtrago( + dependencies=[create_gas_polygons_statusquo, power_plants] + ) + + # Fill eTraGo generators tables + fill_etrago_generators = Egon_etrago_gen( + dependencies=[power_plants, weather_data] + ) + + # Heat supply + heat_supply = HeatSupply( + dependencies=[ + chp, + data_bundle, + district_heating_areas, + zensus_mv_grid_districts, + scenario_capacities, + ] + ) + + + # Pumped hydro and home storage units + storage_units = Storages( + dependencies=[ + mastr_data, + mv_grid_districts, + power_plants, + scenario_parameters, + setup, + vg250_mv_grid_districts, + fill_etrago_generators, + ] + ) + + # Heat to eTraGo + heat_etrago = HeatEtrago( + dependencies=[ + heat_supply, + mv_grid_districts, + renewable_feedin, + setup_etrago, + heat_time_series, + ] + ) + + # CHP to eTraGo + chp_etrago = ChpEtrago(dependencies=[chp, heat_etrago]) + + # Heat pump disaggregation for status quo + heat_pumps_sq = HeatPumpsStatusQuo( + dependencies=[ + cts_demand_buildings, + DistrictHeatingAreas, + heat_supply, + heat_time_series, + power_plants, + ] + ) + + # HTS to eTraGo table + hts_etrago_table = HtsEtragoTable( + dependencies=[ + district_heating_areas, + heat_etrago, + heat_time_series, + mv_grid_districts, + heat_pumps_sq, + ] + ) + + # Storages to eTraGo + storage_etrago = StorageEtrago( + dependencies=[storage_units, scenario_parameters, setup_etrago] + ) + + # eMobility: motorized individual travel TODO: adjust for SQ + emobility_mit = MotorizedIndividualTravel( + dependencies=[ + data_bundle, + mv_grid_districts, + scenario_parameters, + setup_etrago, + zensus_mv_grid_districts, + zensus_vg250, + storage_etrago, + hts_etrago_table, + chp_etrago, + heat_etrago, + fill_etrago_generators, + create_ocgt, + gas_production_insert_data, + insert_data_ch4_storages, + ] + ) + + mit_charging_infrastructure = MITChargingInfrastructure( + dependencies=[mv_grid_districts, hh_demand_buildings_setup] + ) + + # Create load areas + load_areas = LoadArea( + dependencies=[ + osm_landuse, + zensus_vg250, + household_electricity_demand_annual, + tasks[ + "electricity_demand_timeseries" + ".hh_buildings" + ".get-building-peak-loads" + ], + cts_demand_buildings, + demand_curves_industry, + ] + ) + + diff --git a/src/egon/data/airflow/webserver_config.py b/src/egon/data/airflow/webserver_config.py new file mode 100644 index 000000000..075331d0a --- /dev/null +++ b/src/egon/data/airflow/webserver_config.py @@ -0,0 +1,131 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Default configuration for the Airflow webserver.""" +from __future__ import annotations + +import os + +from airflow.www.fab_security.manager import AUTH_DB + +# from airflow.www.fab_security.manager import AUTH_LDAP +# from airflow.www.fab_security.manager import AUTH_OAUTH +# from airflow.www.fab_security.manager import AUTH_OID +# from airflow.www.fab_security.manager import AUTH_REMOTE_USER + + +basedir = os.path.abspath(os.path.dirname(__file__)) + +# Flask-WTF flag for CSRF +WTF_CSRF_ENABLED = True +WTF_CSRF_TIME_LIMIT = None + +# ---------------------------------------------------- +# AUTHENTICATION CONFIG +# ---------------------------------------------------- +# For details on how to set up each of the following authentication, see +# http://flask-appbuilder.readthedocs.io/en/latest/security.html# authentication-methods +# for details. + +# The authentication type +# AUTH_OID : Is for OpenID +# AUTH_DB : Is for database +# AUTH_LDAP : Is for LDAP +# AUTH_REMOTE_USER : Is for using REMOTE_USER from web server +# AUTH_OAUTH : Is for OAuth +AUTH_TYPE = AUTH_DB + +# Uncomment to setup Full admin role name +# AUTH_ROLE_ADMIN = 'Admin' + +# Uncomment and set to desired role to enable access without authentication +AUTH_ROLE_PUBLIC = 'Admin' + +# Will allow user self registration +# AUTH_USER_REGISTRATION = True + +# The recaptcha it's automatically enabled for user self registration is active and the keys are necessary +# RECAPTCHA_PRIVATE_KEY = PRIVATE_KEY +# RECAPTCHA_PUBLIC_KEY = PUBLIC_KEY + +# Config for Flask-Mail necessary for user self registration +# MAIL_SERVER = 'smtp.gmail.com' +# MAIL_USE_TLS = True +# MAIL_USERNAME = 'yourappemail@gmail.com' +# MAIL_PASSWORD = 'passwordformail' +# MAIL_DEFAULT_SENDER = 'sender@gmail.com' + +# The default user self registration role +# AUTH_USER_REGISTRATION_ROLE = "Public" + +# When using OAuth Auth, uncomment to setup provider(s) info +# Google OAuth example: +# OAUTH_PROVIDERS = [{ +# 'name':'google', +# 'token_key':'access_token', +# 'icon':'fa-google', +# 'remote_app': { +# 'api_base_url':'https://www.googleapis.com/oauth2/v2/', +# 'client_kwargs':{ +# 'scope': 'email profile' +# }, +# 'access_token_url':'https://accounts.google.com/o/oauth2/token', +# 'authorize_url':'https://accounts.google.com/o/oauth2/auth', +# 'request_token_url': None, +# 'client_id': GOOGLE_KEY, +# 'client_secret': GOOGLE_SECRET_KEY, +# } +# }] + +# When using LDAP Auth, setup the ldap server +# AUTH_LDAP_SERVER = "ldap://ldapserver.new" + +# When using OpenID Auth, uncomment to setup OpenID providers. +# example for OpenID authentication +# OPENID_PROVIDERS = [ +# { 'name': 'Yahoo', 'url': 'https://me.yahoo.com' }, +# { 'name': 'AOL', 'url': 'http://openid.aol.com/' }, +# { 'name': 'Flickr', 'url': 'http://www.flickr.com/' }, +# { 'name': 'MyOpenID', 'url': 'https://www.myopenid.com' }] + +# ---------------------------------------------------- +# Theme CONFIG +# ---------------------------------------------------- +# Flask App Builder comes up with a number of predefined themes +# that you can use for Apache Airflow. +# http://flask-appbuilder.readthedocs.io/en/latest/customizing.html#changing-themes +# Please make sure to remove "navbar_color" configuration from airflow.cfg +# in order to fully utilize the theme. (or use that property in conjunction with theme) +# APP_THEME = "bootstrap-theme.css" # default bootstrap +# APP_THEME = "amelia.css" +# APP_THEME = "cerulean.css" +# APP_THEME = "cosmo.css" +# APP_THEME = "cyborg.css" +# APP_THEME = "darkly.css" +# APP_THEME = "flatly.css" +# APP_THEME = "journal.css" +# APP_THEME = "lumen.css" +# APP_THEME = "paper.css" +# APP_THEME = "readable.css" +# APP_THEME = "sandstone.css" +# APP_THEME = "simplex.css" +# APP_THEME = "slate.css" +# APP_THEME = "solar.css" +# APP_THEME = "spacelab.css" +# APP_THEME = "superhero.css" +# APP_THEME = "united.css" +# APP_THEME = "yeti.css" diff --git a/src/egon/data/cli.py b/src/egon/data/cli.py index eee6ceec3..598b6015c 100644 --- a/src/egon/data/cli.py +++ b/src/egon/data/cli.py @@ -15,6 +15,7 @@ Also see (1) from http://click.pocoo.org/5/setuptools/#setuptools-integration """ import os +import shutil import socket import subprocess import sys @@ -98,6 +99,15 @@ ), show_default=True, ) +@click.option( + "--household-demand-source", + type=click.Choice(["IEE", "demand-regio"]), + default="demand-regio", + help=( + "Choose the source to calculate and allocate household demands." + ), + show_default=True, +) @click.option( "--jobs", default=1, @@ -162,6 +172,17 @@ ), show_default=True, ) + +@click.option( + "--scenarios", + default=["status2023"], + metavar="SCENARIOS", + help=( + "List of scenario names for which a data model shall be created." + ), + show_default=True, +) + @click.version_option(version=egon.data.__version__) @click.pass_context def egon_data(context, **kwargs): @@ -266,16 +287,19 @@ def options(value, check=None): with open(config.paths(pid="*")[0]) as f: options = yaml.load(f, Loader=yaml.SafeLoader) else: # len(config.paths(pid="*")) == 0, so need to create one. + with open(config.paths()[0]) as f: options["file"] = yaml.load(f, Loader=yaml.SafeLoader) + options = dict( options.get("file", {}), **{ - flag: options["cli"][flag] + flag: options["file"][flag] for flag in options["cli"] if options["cli"][flag] != options["defaults"][flag] }, ) + with open(config.paths(pid="current")[0], "w") as f: f.write(yaml.safe_dump(options)) @@ -314,6 +338,12 @@ def render(template, target, update=True, inserts={}, **more_inserts): ) (Path(".") / "docker" / "database-data").mkdir(parents=True, exist_ok=True) + # Copy webserver_config.py to disable authentification on webinterface + shutil.copy2( + os.path.dirname(egon.data.airflow.__file__) + "/webserver_config.py", + Path(".") / "airflow/webserver_config.py", + ) + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: code = s.connect_ex( (options["--database-host"], int(options["--database-port"])) @@ -387,6 +417,7 @@ def render(template, target, update=True, inserts={}, **more_inserts): connection.host = options["--database-host"] connection.port = options["--database-port"] connection.schema = options["--database-name"] + connection.conn_type = "pgsql" airflow.add(connection) airflow.commit() diff --git a/src/egon/data/config.py b/src/egon/data/config.py index b019733e3..e26e87f41 100644 --- a/src/egon/data/config.py +++ b/src/egon/data/config.py @@ -73,6 +73,7 @@ def settings() -> dict[str, dict[str, str]]: "--jobs": 1, "--random-seed": 42, "--processes-per-task": 1, + "--scenarios": "status2023", } } with open(files[0]) as f: diff --git a/src/egon/data/datasets.yml b/src/egon/data/datasets.yml old mode 100755 new mode 100644 index 314747388..db6581154 --- a/src/egon/data/datasets.yml +++ b/src/egon/data/datasets.yml @@ -1,13 +1,13 @@ openstreetmap: original_data: source: - url: "https://download.geofabrik.de/europe/germany-220101.osm.pbf" - url_testmode: "https://download.geofabrik.de/europe/germany/schleswig-holstein-220101.osm.pbf" + url: "https://download.geofabrik.de/europe/germany-240101.osm.pbf" + url_testmode: "https://download.geofabrik.de/europe/germany/schleswig-holstein-240101.osm.pbf" stylefile: "oedb.style" target: table_prefix: "osm" - file: "germany-220101.osm.pbf" - file_testmode: "schleswig-holstein-220101.osm.pbf" + file: "germany-240101.osm.pbf" + file_testmode: "schleswig-holstein-240101.osm.pbf" processed: schema: "openstreetmap" tables: @@ -90,10 +90,21 @@ map_zensus_vg250: demandregio_installation: sources: git-repository: "https://github.com/openego/disaggregator.git" - branch: "features/pip_install" + branch: "features/pandas-update" targets: path: 'demandregio-disaggregator' +demandregio_workaround: + source: + cache: + url: "https://tubcloud.tu-berlin.de/s/dKqF6wKJnLRyDws/download/cache.zip" + dbdump: + url: "https://tubcloud.tu-berlin.de/s/ktaxyo8kSTK8w3f/download/status2019-egon-demandregio-cts-ind.zip" + targets: + cache: + path: 'demandregio-disaggregator/disaggregator/disaggregator/data_in/' + dbdump: + path: "demandregio_dbdump" demandregio_society: sources: disaggregator: @@ -308,8 +319,10 @@ mastr_new: - "storage" - "wind" file_basename: "bnetza_mastr" - deposit_id: 10480958 + deposit_id: 10491882 egon2021_date_max: "2021-12-31 23:59:00" + status2019_date_max: "2019-12-31 23:59:00" + status2023_date_max: "2023-12-31 23:59:00" geocoding_path: - "data_bundle_egon_data" - "mastr_geocoding" @@ -381,7 +394,13 @@ power_plants: nep_conv: "supply.egon_nep_2021_conventional_powerplants" buses_data: "osmtgmod_results.bus_data" power_plants: 'supply.egon_power_plants' + biomass: 'supply.egon_power_plants_biomass' + hydro: 'supply.egon_power_plants_hydro' + pv: 'supply.egon_power_plants_pv' + wind: 'supply.egon_power_plants_wind' nep_2035: "NEP2035_V2021_scnC2035.xlsx" + wind_offshore_status2019: "windoffshore_status2019.xlsx" + storages: 'supply.egon_storages' target: table: 'egon_power_plants' schema: 'supply' @@ -529,6 +548,9 @@ solar_rooftop: federal_states: schema: 'boundaries' table: 'vg250_lan' + power_plants: + schema: 'supply' + table: 'egon_power_plants' targets: generators: schema: 'grid' @@ -702,7 +724,7 @@ distributed_industrial_demand: demandregio: schema: 'demand' table: 'egon_demandregio_cts_ind' - scenarios: ["eGon2021", "eGon2035", "eGon100RE"] + scenarios: ["status2019","eGon2021", "eGon2035", "eGon100RE"] wz: schema: 'demand' table: 'egon_demandregio_wz' @@ -1123,6 +1145,12 @@ emobility_mit: columns: "D, J:N" skiprows: 8 trips: + status2019: + file: "eGon2035_RS7_min2k_2022-06-01_175429_simbev_run.tar.gz" + file_metadata: "metadata_simbev_run.json" + status2023: + file: "eGon2035_RS7_min2k_2022-06-01_175429_simbev_run.tar.gz" + file_metadata: "metadata_simbev_run.json" eGon2035: file: "eGon2035_RS7_min2k_2022-06-01_175429_simbev_run.tar.gz" file_metadata: "metadata_simbev_run.json" @@ -1132,6 +1160,8 @@ emobility_mit: scenario: # used scenario variation (available scenarios see parameters.py) variation: + status2019: "status2019" + status2023: "status2023" eGon2035: "NEP C 2035" eGon100RE: "Reference 2050" # name of low-flex scenario diff --git a/src/egon/data/datasets/DSM_cts_ind.py b/src/egon/data/datasets/DSM_cts_ind.py index a74ae11da..49d8807e6 100644 --- a/src/egon/data/datasets/DSM_cts_ind.py +++ b/src/egon/data/datasets/DSM_cts_ind.py @@ -8,6 +8,10 @@ sanity checks. See [#1120](https://github.com/openego/eGon-data/issues/1120) for updates. """ +import datetime +import json + +from omi.dialects import get_dialect from sqlalchemy import ARRAY, Column, Float, Integer, String from sqlalchemy.ext.declarative import declarative_base import geopandas as gpd @@ -18,6 +22,15 @@ from egon.data.datasets import Dataset from egon.data.datasets.electricity_demand.temporal import calc_load_curve from egon.data.datasets.industry.temporal import identify_bus +from egon.data.metadata import ( + context, + contributors, + generate_resource_fields_from_db_table, + license_odbl, + meta_metadata, + meta_metadata, + sources, +) # CONSTANTS # TODO: move to datasets.yml @@ -128,7 +141,7 @@ def __init__(self, dependencies): name=self.name, version=self.version, dependencies=self.dependencies, - tasks=(dsm_cts_ind_processing), + tasks=(dsm_cts_ind_processing,), ) @@ -205,6 +218,195 @@ class EgonSitesIndLoadCurvesIndividualDsmTimeseries(Base): e_min = Column(ARRAY(Float)) +def add_metadata_individual(): + targets = config.datasets()["DSM_CTS_industry"]["targets"] + + targets = { + k: v for k, v in targets.items() if "dsm_timeseries" in v["table"] + } + + title_dict = { + "egon_etrago_electricity_cts_dsm_timeseries": ( + "DSM flexibility band time series for CTS" + ), + "egon_osm_ind_load_curves_individual_dsm_timeseries": ( + "DSM flexibility band time series for OSM industry sites" + ), + "egon_demandregio_sites_ind_electricity_dsm_timeseries": ( + "DSM flexibility band time series for demandregio industry sites" + ), + "egon_sites_ind_load_curves_individual_dsm_timeseries": ( + "DSM flexibility band time series for other industry sites" + ), + } + + description_dict = { + "egon_etrago_electricity_cts_dsm_timeseries": ( + "DSM flexibility band time series for CTS in 1 h resolution " + "including available store capacity and power potential" + ), + "egon_osm_ind_load_curves_individual_dsm_timeseries": ( + "DSM flexibility band time series for OSM industry sites in 1 h " + "resolution including available store capacity and power potential" + ), + "egon_demandregio_sites_ind_electricity_dsm_timeseries": ( + "DSM flexibility band time series for demandregio industry sites " + "in 1 h resolution including available store capacity and power " + "potential" + ), + "egon_sites_ind_load_curves_individual_dsm_timeseries": ( + "DSM flexibility band time series for other industry sites in 1 h " + "resolution including available store capacity and power potential" + ), + } + + keywords_dict = { + "egon_etrago_electricity_cts_dsm_timeseries": ["cts"], + "egon_osm_ind_load_curves_individual_dsm_timeseries": [ + "osm", + "industry", + ], + "egon_demandregio_sites_ind_electricity_dsm_timeseries": [ + "demandregio", + "industry", + ], + "egon_sites_ind_load_curves_individual_dsm_timeseries": ["industry"], + } + + primaryKey_dict = { + "egon_etrago_electricity_cts_dsm_timeseries": ["bus"], + "egon_osm_ind_load_curves_individual_dsm_timeseries": ["osm_id"], + "egon_demandregio_sites_ind_electricity_dsm_timeseries": [ + "industrial_sites_id", + ], + "egon_sites_ind_load_curves_individual_dsm_timeseries": ["site_id"], + } + + sources_dict = { + "egon_etrago_electricity_cts_dsm_timeseries": [ + sources()["nep2021"], + sources()["zensus"], + ], + "egon_osm_ind_load_curves_individual_dsm_timeseries": [ + sources()["hotmaps_industrial_sites"], + sources()["schmidt"], + sources()["seenergies"], + ], + "egon_demandregio_sites_ind_electricity_dsm_timeseries": [ + sources()["openstreetmap"], + ], + "egon_sites_ind_load_curves_individual_dsm_timeseries": [ + sources()["hotmaps_industrial_sites"], + sources()["openstreetmap"], + sources()["schmidt"], + sources()["seenergies"], + ], + } + + contris = contributors(["kh", "kh"]) + + contris[0]["date"] = "2023-03-17" + + contris[0]["object"] = "metadata" + contris[1]["object"] = "dataset" + + contris[0]["comment"] = "Add metadata to dataset." + contris[1]["comment"] = "Add workflow to generate dataset." + + for t_dict in targets.values(): + schema = t_dict["schema"] + table = t_dict["table"] + name = f"{schema}.{table}" + + meta = { + "name": name, + "title": title_dict[table], + "id": "WILL_BE_SET_AT_PUBLICATION", + "description": description_dict[table], + "language": "en-US", + "keywords": ["dsm", "timeseries"] + keywords_dict[table], + "publicationDate": datetime.date.today().isoformat(), + "context": context(), + "spatial": { + "location": "none", + "extent": "Germany", + "resolution": "none", + }, + "temporal": { + "referenceDate": "2011-01-01", + "timeseries": { + "start": "2011-01-01", + "end": "2011-12-31", + "resolution": "1 h", + "alignment": "left", + "aggregationType": "average", + }, + }, + "sources": [ + sources()["egon-data"], + sources()["vg250"], + sources()["demandregio"], + ] + + sources_dict[table], + "licenses": [license_odbl("© eGon development team")], + "contributors": contris, + "resources": [ + { + "profile": "tabular-data-resource", + "name": name, + "path": "None", + "format": "PostgreSQL", + "encoding": "UTF-8", + "schema": { + "fields": generate_resource_fields_from_db_table( + schema, + table, + ), + "primaryKey": ["scn_name"] + primaryKey_dict[table], + }, + "dialect": {"delimiter": "", "decimalSeparator": ""}, + } + ], + "review": {"path": "", "badge": ""}, + "metaMetadata": meta_metadata(), + "_comment": { + "metadata": ( + "Metadata documentation and explanation (https://" + "github.com/OpenEnergyPlatform/oemetadata/blob/master/" + "metadata/v141/metadata_key_description.md)" + ), + "dates": ( + "Dates and time must follow the ISO8601 including time " + "zone (YYYY-MM-DD or YYYY-MM-DDThh:mm:ss±hh)" + ), + "units": "Use a space between numbers and units (100 m)", + "languages": ( + "Languages must follow the IETF (BCP47) format (en-GB, " + "en-US, de-DE)" + ), + "licenses": ( + "License name must follow the SPDX License List " + "(https://spdx.org/licenses/)" + ), + "review": ( + "Following the OEP Data Review (https://github.com/" + "OpenEnergyPlatform/data-preprocessing/wiki)" + ), + "none": "If not applicable use (none)", + }, + } + + dialect = get_dialect(f"oep-v{meta_metadata()['metadataVersion'][4:7]}")() + + meta = dialect.compile_and_render(dialect.parse(json.dumps(meta))) + + db.submit_comment( + f"'{json.dumps(meta)}'", + schema, + table, + ) + + # Code def cts_data_import(cts_cool_vent_ac_share): """ @@ -999,10 +1201,10 @@ def delete_dsm_entries(carrier): # buses - sql = f""" - DELETE FROM {targets["bus"]["schema"]}.{targets["bus"]["table"]} b - WHERE (b.carrier LIKE '{carrier}'); - """ + sql = ( + f"DELETE FROM {targets['bus']['schema']}.{targets['bus']['table']} b " + f"WHERE (b.carrier LIKE '{carrier}');" + ) db.execute_sql(sql) # links @@ -1621,3 +1823,5 @@ def dsm_cts_ind_processing(): dsm_cts_ind() dsm_cts_ind_individual() + + add_metadata_individual() diff --git a/src/egon/data/datasets/__init__.py b/src/egon/data/datasets/__init__.py index 3b376b35b..50833f0b0 100644 --- a/src/egon/data/datasets/__init__.py +++ b/src/egon/data/datasets/__init__.py @@ -4,21 +4,33 @@ from collections import abc from dataclasses import dataclass -from functools import reduce +from functools import partial, reduce, update_wrapper from typing import Callable, Iterable, Set, Tuple, Union import re -from airflow.operators import BaseOperator as Operator -from airflow.operators.python_operator import PythonOperator +from airflow.models.baseoperator import BaseOperator as Operator +from airflow.operators.python import PythonOperator from sqlalchemy import Column, ForeignKey, Integer, String, Table, orm, tuple_ from sqlalchemy.ext.declarative import declarative_base -from egon.data import db, logger +from egon.data import config, db, logger Base = declarative_base() SCHEMA = "metadata" +def wrapped_partial(func, *args, **kwargs): + """Like :func:`functools.partial`, but preserves the original function's + name and docstring. Also allows to add a postfix to the function's name. + """ + postfix = kwargs.pop("postfix", None) + partial_func = partial(func, *args, **kwargs) + update_wrapper(partial_func, func) + if postfix: + partial_func.__name__ = f"{func.__name__}{postfix}" + return partial_func + + def setup(): """Create the database structure for storing dataset information.""" # TODO: Move this into a task generating the initial database structure. @@ -66,6 +78,7 @@ class Model(Base): name = Column(String, unique=True, nullable=False) version = Column(String, nullable=False) epoch = Column(Integer, default=0) + scenarios = Column(String, nullable=False) dependencies = orm.relationship( "Model", secondary=DependencyGraph, @@ -186,14 +199,23 @@ class Dataset: tasks: Tasks = () def check_version(self, after_execution=()): + scenario_names = config.settings()["egon-data"]["--scenarios"] + def skip_task(task, *xs, **ks): with db.session_scope() as session: datasets = session.query(Model).filter_by(name=self.name).all() - if self.version in [ - ds.version for ds in datasets - ] and not re.search(r"\.dev$", self.version): + if ( + self.version in [ds.version for ds in datasets] + and scenario_names + == [ + ds.scenarios.replace("{", "").replace("}", "") + for ds in datasets + ] + and not re.search(r"\.dev$", self.version) + ): logger.info( f"Dataset '{self.name}' version '{self.version}'" + f" scenarios {scenario_names}" f" already executed. Skipping." ) else: @@ -207,7 +229,11 @@ def skip_task(task, *xs, **ks): return skip_task def update(self, session): - dataset = Model(name=self.name, version=self.version) + dataset = Model( + name=self.name, + version=self.version, + scenarios=config.settings()["egon-data"]["--scenarios"], + ) dependencies = ( session.query(Model) .filter( diff --git a/src/egon/data/datasets/ch4_prod.py b/src/egon/data/datasets/ch4_prod.py index 786d879ea..9a1affdfc 100755 --- a/src/egon/data/datasets/ch4_prod.py +++ b/src/egon/data/datasets/ch4_prod.py @@ -45,7 +45,7 @@ class CH4Production(Dataset): name: str = "CH4Production" #: - version: str = "0.0.7" + version: str = "0.0.9" def __init__(self, dependencies): super().__init__( @@ -280,7 +280,7 @@ def load_biogas_generators(scn_name): return biogas_generators_list -def import_gas_generators(scn_name="eGon2035"): +def import_gas_generators(): """ Inserts list of gas production units into the database @@ -324,53 +324,76 @@ def import_gas_generators(scn_name="eGon2035"): source = config.datasets()["gas_prod"]["source"] target = config.datasets()["gas_prod"]["target"] - # Clean table - db.execute_sql( - f""" - DELETE FROM {target['stores']['schema']}.{target['stores']['table']} - WHERE "carrier" = 'CH4' AND - scn_name = '{scn_name}' AND bus not IN ( - SELECT bus_id FROM {source['buses']['schema']}.{source['buses']['table']} - WHERE scn_name = '{scn_name}' AND country != 'DE' - ); - """ - ) - - CH4_generators_list = pd.concat( - [load_NG_generators(scn_name), load_biogas_generators(scn_name)] - ) - - # Add missing columns - c = {"scn_name": scn_name, "carrier": "CH4"} - CH4_generators_list = CH4_generators_list.assign(**c) - - # Match to associated CH4 bus - CH4_generators_list = db.assign_gas_bus_id( - CH4_generators_list, scn_name, "CH4" - ) - - # Remove useless columns - CH4_generators_list = CH4_generators_list.drop(columns=["geom", "bus_id"]) - - # Aggregate ch4 productions with same properties at the same bus - CH4_generators_list = ( - CH4_generators_list.groupby( - ["bus", "carrier", "scn_name", "marginal_cost"] + for scn_name in config.settings()["egon-data"]["--scenarios"]: + # Clean table + db.execute_sql( + f""" + DELETE FROM {target['stores']['schema']}.{target['stores']['table']} + WHERE "carrier" = 'CH4' AND + scn_name = '{scn_name}' AND bus not IN ( + SELECT bus_id FROM {source['buses']['schema']}.{source['buses']['table']} + WHERE scn_name = '{scn_name}' AND country != 'DE' + ); + """ ) - .agg({"p_nom": "sum"}) - .reset_index(drop=False) - ) - new_id = db.next_etrago_id("generator") - CH4_generators_list["generator_id"] = range( - new_id, new_id + len(CH4_generators_list) - ) + if scn_name == "eGon2035": + CH4_generators_list = pd.concat( + [ + load_NG_generators(scn_name), + load_biogas_generators(scn_name), + ] + ) + + # Add missing columns + c = {"scn_name": scn_name, "carrier": "CH4"} + CH4_generators_list = CH4_generators_list.assign(**c) + + # Match to associated CH4 bus + CH4_generators_list = db.assign_gas_bus_id( + CH4_generators_list, scn_name, "CH4" + ) + + # Remove useless columns + CH4_generators_list = CH4_generators_list.drop( + columns=["geom", "bus_id"] + ) + + # Aggregate ch4 productions with same properties at the same bus + CH4_generators_list = ( + CH4_generators_list.groupby( + ["bus", "carrier", "scn_name", "marginal_cost"] + ) + .agg({"p_nom": "sum"}) + .reset_index(drop=False) + ) + + elif "status" in scn_name: + # Add one large CH4 generator at each CH4 bus + CH4_generators_list = db.select_dataframe( + f""" + SELECT bus_id as bus, scn_name, carrier + FROM grid.egon_gas_voronoi + WHERE scn_name = '{scn_name}' + AND carrier = 'CH4' + """ + ) + + CH4_generators_list["marginal_cost"] = get_sector_parameters( + "gas", scn_name + )["marginal_cost"]["CH4"] + CH4_generators_list["p_nom"] = 100000 + + new_id = db.next_etrago_id("generator") + CH4_generators_list["generator_id"] = range( + new_id, new_id + len(CH4_generators_list) + ) - # Insert data to db - CH4_generators_list.to_sql( - target["stores"]["table"], - engine, - schema=target["stores"]["schema"], - index=False, - if_exists="append", - ) + # Insert data to db + CH4_generators_list.to_sql( + target["stores"]["table"], + engine, + schema=target["stores"]["schema"], + index=False, + if_exists="append", + ) diff --git a/src/egon/data/datasets/ch4_storages.py b/src/egon/data/datasets/ch4_storages.py index 18e1db935..be0674ef4 100755 --- a/src/egon/data/datasets/ch4_storages.py +++ b/src/egon/data/datasets/ch4_storages.py @@ -54,10 +54,15 @@ def __init__(self, dependencies): name=self.name, version=self.version, dependencies=dependencies, - tasks=(insert_ch4_storages), + # tasks=(insert_ch4_storages), + tasks=(notasks), ) +def notasks(): + return None + + def import_installed_ch4_storages(scn_name): """ Defines list of CH4 stores from the SciGRID_gas data diff --git a/src/egon/data/datasets/chp/__init__.py b/src/egon/data/datasets/chp/__init__.py index 1ff26fedc..c96dcb31a 100644 --- a/src/egon/data/datasets/chp/__init__.py +++ b/src/egon/data/datasets/chp/__init__.py @@ -2,8 +2,10 @@ The central module containing all code dealing with combined heat and power (CHP) plants. """ - from pathlib import Path +import datetime +import json +import time from geoalchemy2 import Geometry from shapely.ops import nearest_points @@ -16,8 +18,8 @@ import pypsa from egon.data import config, db -from egon.data.datasets import Dataset -from egon.data.datasets.chp.match_nep import insert_large_chp +from egon.data.datasets import Dataset, wrapped_partial +from egon.data.datasets.chp.match_nep import insert_large_chp, map_carrier from egon.data.datasets.chp.small_chp import ( assign_use_case, existing_chp_smaller_10mw, @@ -25,13 +27,23 @@ extension_to_areas, select_target, ) -from egon.data.datasets.mastr import WORKING_DIR_MASTR_OLD +from egon.data.datasets.mastr import ( + WORKING_DIR_MASTR_OLD, + WORKING_DIR_MASTR_NEW, +) from egon.data.datasets.power_plants import ( assign_bus_id, assign_voltage_level, filter_mastr_geometry, scale_prox2now, ) +from egon.data.metadata import ( + context, + generate_resource_fields_from_sqla_model, + license_egon_data_odbl, + meta_metadata, + sources, +) Base = declarative_base() @@ -67,6 +79,106 @@ class EgonMaStRConventinalWithoutChp(Base): geometry = Column(Geometry("POINT", 4326)) +def metadata(): + """Write metadata for heat supply tables + + Returns + ------- + None. + + """ + + fields = generate_resource_fields_from_sqla_model(EgonChp) + + fields_df = pd.DataFrame(data=fields).set_index("name") + fields_df.loc["id", "description"] = "Unique identifyer" + fields_df.loc["sources", "description"] = "List of sources" + fields_df.loc[ + "source_id", "description" + ] = "Names of sources, e.g. MaStr_id" + fields_df.loc["carrier", "description"] = "Energy carrier" + fields_df.loc[ + "district_heating", "description" + ] = "Used in district heating or not" + fields_df.loc[ + "el_capacity", "description" + ] = "Installed electrical capacity" + fields_df.loc["th_capacity", "description"] = "Installed thermal capacity" + fields_df.loc[ + "electrical_bus_id", "description" + ] = "Index of corresponding electricity bus" + fields_df.loc[ + "district_heating_area_id", "description" + ] = "Index of corresponding district heating bus" + fields_df.loc[ + "ch4_bus_id", "description" + ] = "Index of corresponding methane bus" + fields_df.loc["voltage_level", "description"] = "Voltage level" + fields_df.loc["scenario", "description"] = "Name of scenario" + fields_df.loc["geom", "description"] = "Location of CHP plant" + + fields_df.loc["el_capacity", "unit"] = "MW_el" + fields_df.loc["th_capacity", "unit"] = "MW_th" + fields_df.unit.fillna("none", inplace=True) + + fields = fields_df.reset_index().to_dict(orient="records") + + meta_district = { + "name": "supply.egon_chp_plants", + "title": "eGon combined heat and power plants", + "id": "WILL_BE_SET_AT_PUBLICATION", + "description": "Combined heat and power plants", + "language": ["EN"], + "publicationDate": datetime.date.today().isoformat(), + "context": context(), + "spatial": { + "location": None, + "extent": "Germany", + "resolution": None, + }, + "sources": [ + sources()["vg250"], + sources()["egon-data"], + sources()["egon-data_bundle"], + sources()["openstreetmap"], + sources()["mastr"], + ], + "licenses": [license_egon_data_odbl()], + "contributors": [ + { + "title": "Clara Büttner", + "email": "http://github.com/ClaraBuettner", + "date": time.strftime("%Y-%m-%d"), + "object": None, + "comment": "Imported data", + }, + ], + "resources": [ + { + "profile": "tabular-data-resource", + "name": "supply.egon_chp_plants", + "path": None, + "format": "PostgreSQL", + "encoding": "UTF-8", + "schema": { + "fields": fields, + "primaryKey": ["index"], + "foreignKeys": [], + }, + "dialect": {"delimiter": None, "decimalSeparator": "."}, + } + ], + "metaMetadata": meta_metadata(), + } + + # Add metadata as a comment to the table + db.submit_comment( + "'" + json.dumps(meta_district) + "'", + EgonChp.__table__.schema, + EgonChp.__table__.name, + ) + + def create_tables(): """Create tables for chp data Returns @@ -132,7 +244,7 @@ def nearest( return value -def assign_heat_bus(scenario="eGon2035"): +def assign_heat_bus(): """Selects heat_bus for chps used in district heating. Parameters @@ -148,87 +260,88 @@ def assign_heat_bus(scenario="eGon2035"): sources = config.datasets()["chp_location"]["sources"] target = config.datasets()["chp_location"]["targets"]["chp_table"] - # Select CHP with use_case = 'district_heating' - chp = db.select_geodataframe( - f""" - SELECT * FROM - {target['schema']}.{target['table']} - WHERE scenario = '{scenario}' - AND district_heating = True - """, - index_col="id", - epsg=4326, - ) + for scenario in config.settings()["egon-data"]["--scenarios"]: + # Select CHP with use_case = 'district_heating' + chp = db.select_geodataframe( + f""" + SELECT * FROM + {target['schema']}.{target['table']} + WHERE scenario = '{scenario}' + AND district_heating = True + """, + index_col="id", + epsg=4326, + ) - # Select district heating areas and their centroid - district_heating = db.select_geodataframe( - f""" - SELECT area_id, ST_Centroid(geom_polygon) as geom - FROM - {sources['district_heating_areas']['schema']}. - {sources['district_heating_areas']['table']} - WHERE scenario = '{scenario}' - """, - epsg=4326, - ) + # Select district heating areas and their centroid + district_heating = db.select_geodataframe( + f""" + SELECT area_id, ST_Centroid(geom_polygon) as geom + FROM + {sources['district_heating_areas']['schema']}. + {sources['district_heating_areas']['table']} + WHERE scenario = '{scenario}' + """, + epsg=4326, + ) - # Assign district heating area_id to district_heating_chp - # According to nearest centroid of district heating area - chp["district_heating_area_id"] = chp.apply( - nearest, - df=district_heating, - row_geom_col="geom", - df_geom_col="geom", - centroid=True, - src_column="area_id", - axis=1, - ) + # Assign district heating area_id to district_heating_chp + # According to nearest centroid of district heating area + chp["district_heating_area_id"] = chp.apply( + nearest, + df=district_heating, + row_geom_col="geom", + df_geom_col="geom", + centroid=True, + src_column="area_id", + axis=1, + ) - # Drop district heating CHP without heat_bus_id - db.execute_sql( - f""" - DELETE FROM {target['schema']}.{target['table']} - WHERE scenario = '{scenario}' - AND district_heating = True - """ - ) + # Drop district heating CHP without heat_bus_id + db.execute_sql( + f""" + DELETE FROM {target['schema']}.{target['table']} + WHERE scenario = '{scenario}' + AND district_heating = True + """ + ) - # Insert district heating CHP with heat_bus_id - session = sessionmaker(bind=db.engine())() - for i, row in chp.iterrows(): - if row.carrier != "biomass": - entry = EgonChp( - id=i, - sources=row.sources, - source_id=row.source_id, - carrier=row.carrier, - el_capacity=row.el_capacity, - th_capacity=row.th_capacity, - electrical_bus_id=row.electrical_bus_id, - ch4_bus_id=row.ch4_bus_id, - district_heating_area_id=row.district_heating_area_id, - district_heating=row.district_heating, - voltage_level=row.voltage_level, - scenario=scenario, - geom=f"SRID=4326;POINT({row.geom.x} {row.geom.y})", - ) - else: - entry = EgonChp( - id=i, - sources=row.sources, - source_id=row.source_id, - carrier=row.carrier, - el_capacity=row.el_capacity, - th_capacity=row.th_capacity, - electrical_bus_id=row.electrical_bus_id, - district_heating_area_id=row.district_heating_area_id, - district_heating=row.district_heating, - voltage_level=row.voltage_level, - scenario=scenario, - geom=f"SRID=4326;POINT({row.geom.x} {row.geom.y})", - ) - session.add(entry) - session.commit() + # Insert district heating CHP with heat_bus_id + session = sessionmaker(bind=db.engine())() + for i, row in chp.iterrows(): + if row.carrier != "biomass": + entry = EgonChp( + id=i, + sources=row.sources, + source_id=row.source_id, + carrier=row.carrier, + el_capacity=row.el_capacity, + th_capacity=row.th_capacity, + electrical_bus_id=row.electrical_bus_id, + ch4_bus_id=row.ch4_bus_id, + district_heating_area_id=row.district_heating_area_id, + district_heating=row.district_heating, + voltage_level=row.voltage_level, + scenario=scenario, + geom=f"SRID=4326;POINT({row.geom.x} {row.geom.y})", + ) + else: + entry = EgonChp( + id=i, + sources=row.sources, + source_id=row.source_id, + carrier=row.carrier, + el_capacity=row.el_capacity, + th_capacity=row.th_capacity, + electrical_bus_id=row.electrical_bus_id, + district_heating_area_id=row.district_heating_area_id, + district_heating=row.district_heating, + voltage_level=row.voltage_level, + scenario=scenario, + geom=f"SRID=4326;POINT({row.geom.x} {row.geom.y})", + ) + session.add(entry) + session.commit() def insert_biomass_chp(scenario): @@ -284,7 +397,7 @@ def insert_biomass_chp(scenario): mastr_loc, cfg, WORKING_DIR_MASTR_OLD ) mastr_loc = assign_bus_id(mastr_loc, cfg) - mastr_loc = assign_use_case(mastr_loc, cfg["sources"]) + mastr_loc = assign_use_case(mastr_loc, cfg["sources"], scenario) # Insert entries with location session = sessionmaker(bind=db.engine())() @@ -310,6 +423,145 @@ def insert_biomass_chp(scenario): session.commit() +def insert_chp_statusquo(scn="status2019"): + cfg = config.datasets()["chp_location"] + + # import data for MaStR + mastr = pd.read_csv( + WORKING_DIR_MASTR_NEW / "bnetza_mastr_combustion_cleaned.csv" + ) + + mastr_biomass = pd.read_csv( + WORKING_DIR_MASTR_NEW / "bnetza_mastr_biomass_cleaned.csv" + ) + + mastr = pd.concat([mastr, mastr_biomass]).reset_index(drop=True) + + mastr = mastr.loc[mastr.ThermischeNutzleistung > 0] + + mastr = mastr.loc[ + mastr.Energietraeger.isin( + [ + "Erdgas", + "Mineralölprodukte", + "andere Gase", + "nicht biogener Abfall", + "Braunkohle", + "Steinkohle", + "Biomasse", + ] + ) + ] + + mastr.Inbetriebnahmedatum = pd.to_datetime(mastr.Inbetriebnahmedatum) + mastr.DatumEndgueltigeStilllegung = pd.to_datetime( + mastr.DatumEndgueltigeStilllegung + ) + mastr = mastr.loc[ + mastr.Inbetriebnahmedatum + <= config.datasets()["mastr_new"][f"{scn}_date_max"] + ] + + mastr = mastr.loc[ + ( + mastr.DatumEndgueltigeStilllegung + >= config.datasets()["mastr_new"][f"{scn}_date_max"] + ) + | (mastr.DatumEndgueltigeStilllegung.isnull()) + ] + + mastr.groupby("Energietraeger").Nettonennleistung.sum().mul(1e-6) + + geom_municipalities = db.select_geodataframe( + """ + SELECT gen, ST_UNION(geometry) as geom + FROM boundaries.vg250_gem + GROUP BY gen + """ + ).set_index("gen") + + # Assing Laengengrad and Breitengrad to chps without location data + # based on the centroid of the municipaltiy + idx_no_location = mastr[ + (mastr.Laengengrad.isnull()) + & (mastr.Gemeinde.isin(geom_municipalities.index)) + ].index + + mastr.loc[idx_no_location, "Laengengrad"] = ( + geom_municipalities.to_crs(epsg="4326").centroid.x.loc[ + mastr.Gemeinde[idx_no_location] + ] + ).values + + mastr.loc[idx_no_location, "Breitengrad"] = ( + geom_municipalities.to_crs(epsg="4326").centroid.y.loc[ + mastr.Gemeinde[idx_no_location] + ] + ).values + + if ( + config.settings()["egon-data"]["--dataset-boundary"] + == "Schleswig-Holstein" + ): + dropped_capacity = mastr[ + (mastr.Laengengrad.isnull()) + & (mastr.Bundesland == "SchleswigHolstein") + ].Nettonennleistung.sum() + + else: + dropped_capacity = mastr[ + (mastr.Laengengrad.isnull()) + ].Nettonennleistung.sum() + + print( + f""" + CHPs with a total installed electrical capacity of {dropped_capacity} kW are dropped + because of missing or wrong location data + """ + ) + + mastr = mastr[~mastr.Laengengrad.isnull()] + mastr = filter_mastr_geometry(mastr).set_geometry("geometry") + + # Assign bus_id + if len(mastr) > 0: + mastr["voltage_level"] = assign_voltage_level( + mastr, cfg, WORKING_DIR_MASTR_NEW + ) + + gas_bus_id = db.assign_gas_bus_id(mastr, scn, "CH4").bus + + mastr = assign_bus_id(mastr, cfg, drop_missing=True) + + mastr["gas_bus_id"] = gas_bus_id + + mastr = assign_use_case(mastr, cfg["sources"], scn) + + # Insert entries with location + session = sessionmaker(bind=db.engine())() + for i, row in mastr.iterrows(): + if row.ThermischeNutzleistung > 0: + entry = EgonChp( + sources={ + "chp": "MaStR", + "el_capacity": "MaStR", + "th_capacity": "MaStR", + }, + source_id={"MastrNummer": row.EinheitMastrNummer}, + carrier=map_carrier().loc[row.Energietraeger], + el_capacity=row.Nettonennleistung / 1000, + th_capacity=row.ThermischeNutzleistung / 1000, + scenario=scn, + district_heating=row.district_heating, + electrical_bus_id=row.bus_id, + ch4_bus_id=row.gas_bus_id, + voltage_level=row.voltage_level, + geom=f"SRID=4326;POINT({row.Laengengrad} {row.Breitengrad})", + ) + session.add(entry) + session.commit() + + def insert_chp_egon2035(): """Insert CHP plants for eGon2035 considering NEP and MaStR data @@ -501,31 +753,61 @@ def insert_chp_egon100re(): ) -# Add one task per federal state for small CHP extension -if ( - config.settings()["egon-data"]["--dataset-boundary"] - == "Schleswig-Holstein" -): - extension = extension_SH -else: - extension = { - extension_BW, - extension_BY, - extension_HB, - extension_BB, - extension_HE, - extension_MV, - extension_NS, - extension_NW, - extension_SH, - extension_HH, - extension_RP, - extension_SL, - extension_SN, - extension_ST, - extension_TH, - extension_BE, - } +tasks = (create_tables,) + +insert_per_scenario = set() + +if "status2019" in config.settings()["egon-data"]["--scenarios"]: + insert_per_scenario.add( + wrapped_partial(insert_chp_statusquo, scn="status2019", postfix="_2019") + ) + +if "status2023" in config.settings()["egon-data"]["--scenarios"]: + insert_per_scenario.add( + wrapped_partial(insert_chp_statusquo, scn="status2023", postfix="_2023") + ) + +if "eGon2035" in config.settings()["egon-data"]["--scenarios"]: + insert_per_scenario.add(insert_chp_egon2035) + +if "eGon100RE" in config.settings()["egon-data"]["--scenarios"]: + insert_per_scenario.add(insert_chp_egon100re) + +tasks = tasks + (insert_per_scenario, assign_heat_bus) + +extension = set() + +if "eGon2035" in config.settings()["egon-data"]["--scenarios"]: + # Add one task per federal state for small CHP extension + if ( + config.settings()["egon-data"]["--dataset-boundary"] + == "Schleswig-Holstein" + ): + extension = extension_SH + else: + extension = { + extension_BW, + extension_BY, + extension_HB, + extension_BB, + extension_HE, + extension_MV, + extension_NS, + extension_NW, + extension_SH, + extension_HH, + extension_RP, + extension_SL, + extension_SN, + extension_ST, + extension_TH, + extension_BE, + } + +if extension != set(): + tasks = tasks + (extension,) + +tasks += (metadata,) class Chp(Dataset): @@ -560,17 +842,12 @@ class Chp(Dataset): #: name: str = "Chp" #: - version: str = "0.0.6" + version: str = "0.0.7" def __init__(self, dependencies): super().__init__( name=self.name, version=self.version, dependencies=dependencies, - tasks=( - create_tables, - {insert_chp_egon2035, insert_chp_egon100re}, - assign_heat_bus, - extension, - ), + tasks=tasks, ) diff --git a/src/egon/data/datasets/chp/small_chp.py b/src/egon/data/datasets/chp/small_chp.py index 60f36440f..bade2152c 100755 --- a/src/egon/data/datasets/chp/small_chp.py +++ b/src/egon/data/datasets/chp/small_chp.py @@ -3,6 +3,7 @@ """ from sqlalchemy.orm import sessionmaker import geopandas as gpd +import pandas as pd import numpy as np from egon.data import config, db @@ -95,7 +96,7 @@ def existing_chp_smaller_10mw(sources, MaStR_konv, EgonChp): mastr_chp, config.datasets()["chp_location"] ).bus_id - mastr_chp = assign_use_case(mastr_chp, sources) + mastr_chp = assign_use_case(mastr_chp, sources, "eGon2035") insert_mastr_chp(mastr_chp, EgonChp) @@ -163,7 +164,6 @@ def extension_to_areas( # Add new CHP as long as the additional capacity is not reached while additional_capacity > existing_chp.el_capacity.min(): - if district_heating: selected_areas = areas.loc[ areas.demand > existing_chp.th_capacity.min() * flh, : @@ -174,7 +174,6 @@ def extension_to_areas( ] if len(selected_areas) > 0: - selected_areas = selected_areas.to_crs(4326) # Assign gas bus_id selected_areas["gas_bus_id"] = db.assign_gas_bus_id( @@ -378,9 +377,11 @@ def extension_district_heating( if not areas_without_chp_only: # Append district heating areas with CHP # assumed dispatch of existing CHP is substracted from remaining demand - dh_areas = dh_areas.append( - db.select_geodataframe( - f""" + dh_areas = pd.concat( + [ + dh_areas, + db.select_geodataframe( + f""" SELECT b.residential_and_service_demand - sum(a.el_capacity)*{flh_chp} as demand, b.area_id, @@ -403,8 +404,9 @@ def extension_district_heating( b.residential_and_service_demand, b.area_id, geom_polygon) """, - epsg=3035, - ), + epsg=3035, + ), + ], ignore_index=True, ).set_crs(3035, allow_override=True) @@ -573,7 +575,6 @@ def extension_per_federal_state(federal_state, EgonChp): ) if additional_capacity > 0: - share_dh = ( existing_capacity[ existing_capacity.district_heating @@ -631,7 +632,7 @@ def extension_per_federal_state(federal_state, EgonChp): print("Decommissioning of CHP plants is not implemented.") -def assign_use_case(chp, sources): +def assign_use_case(chp, sources, scenario): """Identifies CHPs used in district heating areas. A CHP plant is assigned to a district heating area if @@ -699,7 +700,7 @@ def assign_use_case(chp, sources): SELECT area_id, ST_Buffer(geom_polygon, 1000) as geom FROM {sources['district_heating_areas']['schema']}. {sources['district_heating_areas']['table']} - WHERE scenario = 'eGon2035' + WHERE scenario = '{scenario}' """, epsg=4326, ) @@ -730,8 +731,11 @@ def assign_use_case(chp, sources): # Chp which are close to a district heating area and not close to an # industrial location are assigned as district_heating_chp - district_heating_chp = district_heating_chp.append( - close_to_dh[~close_to_dh.index.isin(close_to_industry.index)] + district_heating_chp = pd.concat( + [ + district_heating_chp, + close_to_dh[~close_to_dh.index.isin(close_to_industry.index)], + ] ) # Set district_heating = True for all district heating chp diff --git a/src/egon/data/datasets/chp_etrago.py b/src/egon/data/datasets/chp_etrago.py index 9a178ce13..19ae3ea0d 100644 --- a/src/egon/data/datasets/chp_etrago.py +++ b/src/egon/data/datasets/chp_etrago.py @@ -45,15 +45,6 @@ def __init__(self, dependencies): def insert_egon100re(): - """Insert combined heat and power plants into eTraGo tables - for the eGon100RE scenario. - - Returns - ------- - None. - - """ - sources = config.datasets()["chp_etrago"]["sources"] targets = config.datasets()["chp_etrago"]["targets"] @@ -159,18 +150,7 @@ def insert_egon100re(): ) -def insert(): - """Insert combined heat and power plants into eTraGo tables. - - Gas CHP plants are modeled as links to the gas grid, - biomass CHP plants (only in eGon2035) are modeled as generators - - Returns - ------- - None. - - """ - +def insert_scenario(scenario): sources = config.datasets()["chp_etrago"]["sources"] targets = config.datasets()["chp_etrago"]["targets"] @@ -179,16 +159,16 @@ def insert(): f""" DELETE FROM {targets['link']['schema']}.{targets['link']['table']} WHERE carrier LIKE '%%CHP%%' - AND scn_name = 'eGon2035' + AND scn_name = '{scenario}' AND bus0 IN (SELECT bus_id FROM {sources['etrago_buses']['schema']}.{sources['etrago_buses']['table']} - WHERE scn_name = 'eGon2035' + WHERE scn_name = '{scenario}' AND country = 'DE') AND bus1 IN (SELECT bus_id FROM {sources['etrago_buses']['schema']}.{sources['etrago_buses']['table']} - WHERE scn_name = 'eGon2035' + WHERE scn_name = '{scenario}' AND country = 'DE') """ ) @@ -196,7 +176,7 @@ def insert(): f""" DELETE FROM {targets['generator']['schema']}.{targets['generator']['table']} WHERE carrier LIKE '%%CHP%%' - AND scn_name = 'eGon2035' + AND scn_name = '{scenario}' """ ) # Select all CHP plants used in district heating @@ -213,9 +193,9 @@ def insert(): JOIN grid.egon_etrago_bus c ON ST_Transform(ST_Centroid(b.geom_polygon), 4326) = c.geom - WHERE a.scenario='eGon2035' - AND b.scenario = 'eGon2035' - AND c.scn_name = 'eGon2035' + WHERE a.scenario='{scenario}' + AND b.scenario = '{scenario}' + AND c.scn_name = '{scenario}' AND c.carrier = 'central_heat' AND NOT district_heating_area_id IS NULL GROUP BY ( @@ -223,15 +203,15 @@ def insert(): """ ) # Divide into biomass and gas CHP which are modelled differently - chp_link_dh = chp_dh[chp_dh.carrier != "biomass"].index - chp_generator_dh = chp_dh[chp_dh.carrier == "biomass"].index + chp_link_dh = chp_dh[chp_dh.carrier == "gas"].index + chp_generator_dh = chp_dh[chp_dh.carrier != "gas"].index # Create geodataframes for gas CHP plants chp_el = link_geom_from_buses( gpd.GeoDataFrame( index=chp_link_dh, data={ - "scn_name": "eGon2035", + "scn_name": scenario, "bus0": chp_dh.loc[chp_link_dh, "ch4_bus_id"].astype(int), "bus1": chp_dh.loc[chp_link_dh, "electrical_bus_id"].astype( int @@ -240,7 +220,7 @@ def insert(): "carrier": "central_gas_CHP", }, ), - "eGon2035", + scenario, ) # Set index chp_el["link_id"] = range( @@ -248,7 +228,7 @@ def insert(): ) # Add marginal cost which is only VOM in case of gas chp - chp_el["marginal_cost"] = get_sector_parameters("gas", "eGon2035")[ + chp_el["marginal_cost"] = get_sector_parameters("gas", scenario)[ "marginal_cost" ]["chp_gas"] @@ -265,14 +245,14 @@ def insert(): gpd.GeoDataFrame( index=chp_link_dh, data={ - "scn_name": "eGon2035", + "scn_name": scenario, "bus0": chp_dh.loc[chp_link_dh, "ch4_bus_id"].astype(int), "bus1": chp_dh.loc[chp_link_dh, "heat_bus_id"].astype(int), "p_nom": chp_dh.loc[chp_link_dh, "th_capacity"], "carrier": "central_gas_CHP_heat", }, ), - "eGon2035", + scenario, ) chp_heat["link_id"] = range( @@ -286,17 +266,17 @@ def insert(): if_exists="append", ) - # Insert biomass CHP as generators + # Insert biomass, coal, oil and other CHP as generators # Create geodataframes for CHP plants chp_el_gen = pd.DataFrame( index=chp_generator_dh, data={ - "scn_name": "eGon2035", + "scn_name": scenario, "bus": chp_dh.loc[chp_generator_dh, "electrical_bus_id"].astype( int ), "p_nom": chp_dh.loc[chp_generator_dh, "el_capacity"], - "carrier": "central_biomass_CHP", + "carrier": chp_dh.loc[chp_generator_dh, "carrier"], }, ) @@ -306,9 +286,17 @@ def insert(): ) # Add marginal cost - chp_el_gen["marginal_cost"] = get_sector_parameters( - "electricity", "eGon2035" - )["marginal_cost"]["biomass"] + chp_el_gen["marginal_cost"] = ( + pd.Series( + get_sector_parameters("electricity", scenario)["marginal_cost"] + ) + .rename({"other_non_renewable": "others"}) + .loc[chp_el_gen["carrier"]] + ).values + + chp_el_gen["carrier"] = ( + "central_" + chp_dh.loc[chp_generator_dh, "carrier"] + "_CHP" + ) chp_el_gen.to_sql( targets["generator"]["table"], @@ -321,10 +309,12 @@ def insert(): chp_heat_gen = pd.DataFrame( index=chp_generator_dh, data={ - "scn_name": "eGon2035", + "scn_name": scenario, "bus": chp_dh.loc[chp_generator_dh, "heat_bus_id"].astype(int), "p_nom": chp_dh.loc[chp_generator_dh, "th_capacity"], - "carrier": "central_biomass_CHP_heat", + "carrier": "central_" + + chp_dh.loc[chp_generator_dh, "carrier"] + + "_CHP_heat", }, ) @@ -346,20 +336,20 @@ def insert(): SELECT electrical_bus_id, ch4_bus_id, carrier, SUM(el_capacity) AS el_capacity, SUM(th_capacity) AS th_capacity FROM {sources['chp_table']['schema']}.{sources['chp_table']['table']} - WHERE scenario='eGon2035' + WHERE scenario='{scenario}' AND district_heating_area_id IS NULL GROUP BY (electrical_bus_id, ch4_bus_id, carrier) """ ) - chp_link_ind = chp_industry[chp_industry.carrier != "biomass"].index + chp_link_ind = chp_industry[chp_industry.carrier == "gas"].index - chp_generator_ind = chp_industry[chp_industry.carrier == "biomass"].index + chp_generator_ind = chp_industry[chp_industry.carrier != "gas"].index chp_el_ind = link_geom_from_buses( gpd.GeoDataFrame( index=chp_link_ind, data={ - "scn_name": "eGon2035", + "scn_name": scenario, "bus0": chp_industry.loc[chp_link_ind, "ch4_bus_id"].astype( int ), @@ -370,7 +360,7 @@ def insert(): "carrier": "industrial_gas_CHP", }, ), - "eGon2035", + scenario, ) chp_el_ind["link_id"] = range( @@ -378,7 +368,7 @@ def insert(): ) # Add marginal cost which is only VOM in case of gas chp - chp_el_ind["marginal_cost"] = get_sector_parameters("gas", "eGon2035")[ + chp_el_ind["marginal_cost"] = get_sector_parameters("gas", scenario)[ "marginal_cost" ]["chp_gas"] @@ -393,12 +383,12 @@ def insert(): chp_el_ind_gen = pd.DataFrame( index=chp_generator_ind, data={ - "scn_name": "eGon2035", + "scn_name": scenario, "bus": chp_industry.loc[ chp_generator_ind, "electrical_bus_id" ].astype(int), "p_nom": chp_industry.loc[chp_generator_ind, "el_capacity"], - "carrier": "industrial_biomass_CHP", + "carrier": chp_industry.loc[chp_generator_ind, "carrier"], }, ) @@ -408,9 +398,16 @@ def insert(): ) # Add marginal cost - chp_el_ind_gen["marginal_cost"] = get_sector_parameters( - "electricity", "eGon2035" - )["marginal_cost"]["biomass"] + chp_el_ind_gen["marginal_cost"] = ( + pd.Series( + get_sector_parameters("electricity", scenario)["marginal_cost"] + ) + .rename({"other_non_renewable": "others"}) + .loc[chp_el_ind_gen["carrier"]] + ).values + + # Update carrier + chp_el_ind_gen["carrier"] = "industrial_" + chp_el_ind_gen.carrier + "_CHP" chp_el_ind_gen.to_sql( targets["generator"]["table"], @@ -420,4 +417,22 @@ def insert(): index=False, ) - insert_egon100re() + +def insert(): + """Insert combined heat and power plants into eTraGo tables. + + Gas CHP plants are modeled as links to the gas grid, + biomass CHP plants (only in eGon2035) are modeled as generators + + Returns + ------- + None. + + """ + + for scenario in config.settings()["egon-data"]["--scenarios"]: + if scenario != "eGon100RE": + insert_scenario(scenario) + + else: + insert_egon100re() diff --git a/src/egon/data/datasets/data_bundle/__init__.py b/src/egon/data/datasets/data_bundle/__init__.py index 76db4c650..2258c87b3 100644 --- a/src/egon/data/datasets/data_bundle/__init__.py +++ b/src/egon/data/datasets/data_bundle/__init__.py @@ -1,4 +1,4 @@ -"""The central module containing all code dealing with small scale inpu-data +"""The central module containing all code dealing with small scale input-data """ @@ -13,7 +13,7 @@ def download(): """ - Download small scale imput data from Zenodo + Download small scale input data from Zenodo Parameters ---------- @@ -30,8 +30,10 @@ def download(): ) target_file = config.datasets()["data-bundle"]["targets"]["file"] - # Retrieve files - urlretrieve(url, target_file) + # check if file exists + if not Path(target_file).exists(): + # Retrieve files + urlretrieve(url, target_file) with zipfile.ZipFile(target_file, "r") as zip_ref: zip_ref.extractall(".") @@ -44,7 +46,7 @@ def __init__(self, dependencies): ] super().__init__( name="DataBundle", - version=f"{deposit_id}-0.0.1", + version=f"{deposit_id}-0.0.3", dependencies=dependencies, tasks=(download,), ) diff --git a/src/egon/data/datasets/demandregio/__init__.py b/src/egon/data/datasets/demandregio/__init__.py index 6d8ff5810..a2a837ddc 100644 --- a/src/egon/data/datasets/demandregio/__init__.py +++ b/src/egon/data/datasets/demandregio/__init__.py @@ -3,14 +3,17 @@ """ from pathlib import Path +import subprocess +import os +import zipfile from sqlalchemy import ARRAY, Column, Float, ForeignKey, Integer, String from sqlalchemy.ext.declarative import declarative_base import numpy as np import pandas as pd -from egon.data import db -from egon.data.datasets import Dataset +from egon.data import db, logger +from egon.data.datasets import Dataset, wrapped_partial from egon.data.datasets.demandregio.install_disaggregator import ( clone_and_install, ) @@ -18,11 +21,12 @@ EgonScenario, get_sector_parameters, ) +from egon.data.datasets.zensus import download_and_check import egon.data.config import egon.data.datasets.scenario_parameters.parameters as scenario_parameters try: - from disaggregator import config, data, spatial + from disaggregator import config, data, spatial, temporal except ImportError as e: pass @@ -35,10 +39,11 @@ class DemandRegio(Dataset): def __init__(self, dependencies): super().__init__( name="DemandRegio", - version="0.0.5", + version="0.0.9", dependencies=dependencies, tasks=( clone_and_install, + get_cached_tables, # adhoc workaround #180 create_tables, { insert_household_demand, @@ -48,6 +53,14 @@ def __init__(self, dependencies): ), ) +class DemandRegioLoadProfiles(Base): + __tablename__ = "demandregio_household_load_profiles" + __table_args__ = {"schema": "demand"} + + id = Column(Integer, primary_key=True) + year = Column(Integer) + nuts3 = Column(String) + load_in_mwh = Column(ARRAY(Float())) class EgonDemandRegioHH(Base): __tablename__ = "egon_demandregio_hh" @@ -181,7 +194,6 @@ def insert_cts_ind_wz_definitions(): engine = db.engine() for sector in source["wz_definitions"]: - file_path = ( Path(".") / "data_bundle_egon_data" @@ -424,18 +436,40 @@ def disagg_households_power( pd.DataFrame or pd.Series """ # source: survey of energieAgenturNRW + # with/without direct water heating (DHW), and weighted average + # https://1-stromvergleich.com/wp-content/uploads/erhebung_wo_bleibt_der_strom.pdf demand_per_hh_size = pd.DataFrame( index=range(1, 7), data={ - "weighted DWH": [2290, 3202, 4193, 4955, 5928, 5928], - "without DHW": [1714, 2812, 3704, 4432, 5317, 5317], + # "weighted DWH": [2290, 3202, 4193, 4955, 5928, 5928], + # "without DHW": [1714, 2812, 3704, 4432, 5317, 5317], + "with_DHW": [2181, 3843, 5151, 6189, 7494, 8465], + "without_DHW": [1798, 2850, 3733, 4480, 5311, 5816], + "weighted": [2256, 3248, 4246, 5009, 5969, 6579], }, ) + if scenario == "eGon100RE": + # chose demand per household size from survey without DHW + power_per_HH = ( + demand_per_hh_size["without_DHW"] / 1e3 + ) # TODO why without? + + # calculate demand per nuts3 in 2011 + df_2011 = data.households_per_size(year=2011) * power_per_HH + + # scale demand per hh-size to meet demand without heat + # according to JRC in 2011 (136.6-(20.14+9.41) TWh) + # TODO check source and method + power_per_HH *= (136.6 - (20.14 + 9.41)) * 1e6 / df_2011.sum().sum() + + # calculate demand per nuts3 in 2050 + df = data.households_per_size(year=year) * power_per_HH + # Bottom-Up: Power demand by household sizes in [MWh/a] for each scenario - if scenario in ["eGon2021", "eGon2035"]: + elif scenario in ["status2019", "status2023", "eGon2021", "eGon2035"]: # chose demand per household size from survey including weighted DHW - power_per_HH = demand_per_hh_size["weighted DWH"] / 1e3 + power_per_HH = demand_per_hh_size["weighted"] / 1e3 # calculate demand per nuts3 df = ( @@ -445,22 +479,16 @@ def disagg_households_power( if scenario == "eGon2035": # scale to fit demand of NEP 2021 scebario C 2035 (119TWh) - df *= 119000000 / df.sum().sum() + df *= 119 * 1e6 / df.sum().sum() - elif scenario == "eGon100RE": + if scenario == "status2023": + # scale to fit demand of BDEW 2023 (130.48 TWh) see issue #180 + df *= 130.48 * 1e6 / df.sum().sum() - # chose demand per household size from survey without DHW - power_per_HH = demand_per_hh_size["without DHW"] / 1e3 - - # calculate demand per nuts3 in 2011 - df_2011 = data.households_per_size(year=2011) * power_per_HH - - # scale demand per hh-size to meet demand without heat - # according to JRC in 2011 (136.6-(20.14+9.41) TWh) - power_per_HH *= (136.6 - (20.14 + 9.41)) * 1e6 / df_2011.sum().sum() - - # calculate demand per nuts3 in 2050 - df = data.households_per_size(year=year) * power_per_HH + # if scenario == "status2021": # TODO status2021 + # # scale to fit demand of AGEB 2021 (138.6 TWh) + # # https://ag-energiebilanzen.de/wp-content/uploads/2023/01/AGEB_22p2_rev-1.pdf#page=10 + # df *= 138.6 * 1e6 / df.sum().sum() else: print( @@ -473,6 +501,56 @@ def disagg_households_power( return df +def write_demandregio_hh_profiles_to_db(hh_profiles): + """Write HH demand profiles from demand regio into db. One row per + year and nuts3. The annual load profile timeseries is an array. + + schema: demand + tablename: demandregio_household_load_profiles + + + + Parameters + ---------- + hh_profiles: pd.DataFrame + + Returns + ------- + """ + years = hh_profiles.index.year.unique().values + df_to_db = pd.DataFrame(columns=["id", "year", "nuts3", "load_in_mwh"]).set_index("id") + dataset = egon.data.config.settings()["egon-data"]["--dataset-boundary"] + + if dataset == "Schleswig-Holstein": + hh_profiles = hh_profiles.loc[ + :, hh_profiles.columns.str.contains("DEF0")] + + id = 0 + for year in years: + df = hh_profiles[hh_profiles.index.year == year] + for nuts3 in hh_profiles.columns: + id+=1 + df_to_db.at[id, "year"] = year + df_to_db.at[id, "nuts3"] = nuts3 + df_to_db.at[id, "load_in_mwh"] = df[nuts3].to_list() + + df_to_db["year"] = df_to_db["year"].apply(int) + df_to_db["nuts3"] = df_to_db["nuts3"].astype(str) + df_to_db["load_in_mwh"] = df_to_db["load_in_mwh"].apply(list) + df_to_db = df_to_db.reset_index() + + DemandRegioLoadProfiles.__table__.drop(bind=db.engine(), checkfirst=True) + DemandRegioLoadProfiles.__table__.create(bind=db.engine()) + + df_to_db.to_sql( + name=DemandRegioLoadProfiles.__table__.name, + schema=DemandRegioLoadProfiles.__table__.schema, + con=db.engine(), + if_exists="append", + index=-False, + ) + + return def insert_hh_demand(scenario, year, engine): """Calculates electrical demands of private households using demandregio's @@ -481,7 +559,7 @@ def insert_hh_demand(scenario, year, engine): Parameters ---------- scenario : str - Name of the corresponing scenario. + Name of the corresponding scenario. year : int The number of households per region is taken from this year. @@ -502,7 +580,9 @@ def insert_hh_demand(scenario, year, engine): # insert into database for hh_size in ec_hh.columns: df = pd.DataFrame(ec_hh[hh_size]) - df["year"] = year + df["year"] = 2023 if scenario == "status2023" else year # TODO status2023 + # adhoc fix until ffeopendata servers are up and population_year can be set + df["scenario"] = scenario df["hh_size"] = hh_size df = df.rename({hh_size: "demand"}, axis="columns") @@ -513,6 +593,37 @@ def insert_hh_demand(scenario, year, engine): if_exists="append", ) + # insert housholds demand timeseries + try: + hh_load_timeseries = ( + temporal.disagg_temporal_power_housholds_slp( + use_nuts3code=True, + by="households", + weight_by_income=False, + year=year, + ) + .resample("h") + .sum() + ) + hh_load_timeseries.rename( + columns={"DEB16": "DEB1C", "DEB19": "DEB1D"}, inplace=True) + except Exception as e: + logger.warning(f"Couldnt get profiles from FFE, will use pickeld fallback! \n {e}") + hh_load_timeseries = pd.read_pickle(Path(".", "df_load_profiles.pkl").resolve()) + + def change_year(dt, year): + return dt.replace(year=year) + + year = 2023 if scenario == "status2023" else year # TODO status2023 + hh_load_timeseries.index = hh_load_timeseries.index.map(lambda dt: change_year(dt, year)) + + if scenario == "status2023": + hh_load_timeseries = hh_load_timeseries.shift(24 * 2) + + hh_load_timeseries.iloc[:24 * 7] = hh_load_timeseries.iloc[24 * 7:24 * 7 * 2].values + + write_demandregio_hh_profiles_to_db(hh_load_timeseries) + def insert_cts_ind(scenario, year, engine, target_values): """Calculates electrical demands of CTS and industry using demandregio's @@ -552,7 +663,7 @@ def insert_cts_ind(scenario, year, engine, target_values): # scale values according to target_values if sector in target_values[scenario].keys(): ec_cts_ind *= ( - target_values[scenario][sector] * 1e3 / ec_cts_ind.sum().sum() + target_values[scenario][sector] / ec_cts_ind.sum().sum() ) # include new largescale consumers according to NEP 2021 @@ -595,13 +706,14 @@ def insert_household_demand(): ] engine = db.engine() + scenarios = egon.data.config.settings()["egon-data"]["--scenarios"] + for t in targets: db.execute_sql( f"DELETE FROM {targets[t]['schema']}.{targets[t]['table']};" ) - for scn in ["eGon2021", "eGon2035", "eGon100RE"]: - + for scn in scenarios: year = scenario_parameters.global_settings(scn)["population_year"] # Insert demands of private households @@ -629,8 +741,10 @@ def insert_cts_ind_demands(): insert_cts_ind_wz_definitions() - for scn in ["eGon2021", "eGon2035", "eGon100RE"]: + scenarios = egon.data.config.settings()["egon-data"]["--scenarios"] + + for scn in scenarios: year = scenario_parameters.global_settings(scn)["population_year"] if year > 2035: @@ -640,13 +754,23 @@ def insert_cts_ind_demands(): target_values = { # according to NEP 2021 # new consumers will be added seperatly - "eGon2035": {"CTS": 135300, "industry": 225400}, + "eGon2035": { + "CTS": 135300 * 1e3, + "industry": 225400 * 1e3 + }, # CTS: reduce overall demand from demandregio (without traffic) # by share of heat according to JRC IDEES, data from 2011 # industry: no specific heat demand, use data from demandregio - "eGon100RE": {"CTS": (1 - (5.96 + 6.13) / 154.64) * 125183.403}, + "eGon100RE": { + "CTS": ((1 - (5.96 + 6.13) / 154.64) * 125183.403) * 1e3 + }, # no adjustments for status quo "eGon2021": {}, + "status2019": {}, + "status2023": { + "CTS": 121160 * 1e3, + "industry": 200380 * 1e3 + }, } insert_cts_ind(scn, year, engine, target_values) @@ -727,7 +851,23 @@ def insert_timeseries_per_wz(sector, year): if sector == "CTS": profiles = ( - data.CTS_power_slp_generator("SH", year=year).resample("H").sum() + data.CTS_power_slp_generator("SH", year=year) + .drop( + [ + "Day", + "Hour", + "DayOfYear", + "WD", + "SA", + "SU", + "WIZ", + "SOZ", + "UEZ", + ], + axis="columns", + ) + .resample("H") + .sum() ) wz_slp = config.slp_branch_cts_power() elif sector == "industry": @@ -781,10 +921,26 @@ def timeseries_per_wz(): """ - years = get_sector_parameters("global").weather_year.unique() + scenarios = egon.data.config.settings()["egon-data"]["--scenarios"] - for year in years: + for scn in scenarios: + year = int(scenario_parameters.global_settings(scn)["weather_year"]) for sector in ["CTS", "industry"]: - insert_timeseries_per_wz(sector, int(year)) + + +def get_cached_tables(): + """Get cached demandregio tables and db-dump from former runs""" + data_config = egon.data.config.datasets() + for s in ["cache", "dbdump"]: + url = data_config["demandregio_workaround"]["source"][s]["url"] + target_path = data_config["demandregio_workaround"]["targets"][s]["path"] + filename = os.path.basename(url) + file_path = Path(".", target_path, filename).resolve() + os.makedirs(file_path.parent, exist_ok=True) + logger.info(f"Downloading: {filename} from {url}.") + download_and_check(url, file_path, max_iteration=5) + with zipfile.ZipFile(file_path, "r") as zip_ref: + zip_ref.extractall(file_path.parent) + diff --git a/src/egon/data/datasets/demandregio/install_disaggregator.py b/src/egon/data/datasets/demandregio/install_disaggregator.py index 552242628..fa80831f9 100644 --- a/src/egon/data/datasets/demandregio/install_disaggregator.py +++ b/src/egon/data/datasets/demandregio/install_disaggregator.py @@ -1,16 +1,18 @@ """This module downloads and installs demandregio's disaggregator from GitHub """ +from pathlib import Path +from subprocess import check_output +import importlib.util import os import shutil -from pathlib import Path +from egon.data import logger, subprocess import egon.data.config -from egon.data import subprocess def clone_and_install(): - """ Clone and install repository of demandregio's disaggregator + """Clone and install repository of demandregio's disaggregator Returns ------- @@ -26,27 +28,48 @@ def clone_and_install(): ] ) - # Delete repository if it already exists - if repo_path.exists() and repo_path.is_dir(): - shutil.rmtree(repo_path) - - # Create subfolder - os.mkdir(repo_path) - - # Clone from GitHub repository - subprocess.run( - [ - "git", - "clone", - "--single-branch", - "--branch", - source["branch"], - source["git-repository"], - ], - cwd=repo_path, - ) + try: + status = check_output( + ["git", "status"], cwd=(repo_path / "disaggregator").absolute() + ) + if status.startswith(b"Auf Branch features/pandas-update"): + logger.info("Demandregio cloned and right branch checked out.") + else: + raise ImportError( + "Demandregio cloned but wrong branch checked " + "out. Please checkout features/pandas-update" + ) + spec = importlib.util.find_spec("disaggregator") + if spec is not None: + logger.info("Demandregio is not installed. Installing now.") + # Install disaggregator from path + subprocess.run( + [ + "pip", + "install", + "-e", + (repo_path / "disaggregator").absolute(), + ] + ) - # Install disaggregator from path - subprocess.run( - ["pip", "install", "-e", (repo_path / "disaggregator").absolute()] - ) + except subprocess.CalledProcessError and FileNotFoundError: + # Create subfolder + os.makedirs(repo_path, exist_ok=True) + + # Clone from GitHub repository + subprocess.run( + [ + "git", + "clone", + "--single-branch", + "--branch", + source["branch"], + source["git-repository"], + ], + cwd=repo_path, + ) + + # Install disaggregator from path + subprocess.run( + ["pip", "install", "-e", (repo_path / "disaggregator").absolute()] + ) diff --git a/src/egon/data/datasets/district_heating_areas/__init__.py b/src/egon/data/datasets/district_heating_areas/__init__.py index 660692443..bb548ee4d 100644 --- a/src/egon/data/datasets/district_heating_areas/__init__.py +++ b/src/egon/data/datasets/district_heating_areas/__init__.py @@ -1,4 +1,9 @@ # -*- coding: utf-8 -*- + +# This script is part of eGon-data. + +# license text - to be added. + """ Central module containing all code creating with district heating areas. @@ -7,7 +12,7 @@ end it saves them in the database. """ import os -from egon.data import db +from egon.data import config, db from egon.data.datasets.scenario_parameters import ( get_sector_parameters, EgonScenario, @@ -23,6 +28,14 @@ ) # for metadata creation +import time +import datetime +from egon.data.metadata import ( + context, + meta_metadata, + license_ccby, + sources, +) import json # import time @@ -65,7 +78,7 @@ class DistrictHeatingAreas(Dataset): #: name: str = "district-heating-areas" #: - version: str = "0.0.1" + version: str = "0.0.2" def __init__(self, dependencies): super().__init__( @@ -168,7 +181,7 @@ def create_tables(): # https://geopandas.org/docs/user_guide/geometric_manipulations.html -def load_census_data(): +def load_census_data(minimum_connection_rate=0.3): """ Load the heating type information from the census database table. @@ -248,7 +261,9 @@ def load_census_data(): # district_heat.head # district_heat['connection_rate'].describe() - district_heat = district_heat[district_heat["connection_rate"] >= 0.3] + district_heat = district_heat[ + district_heat["connection_rate"] >= minimum_connection_rate + ] # district_heat.columns return district_heat, heating_type @@ -371,7 +386,7 @@ def area_grouping( # print(cell_buffers.area) # create a shapely Multipolygon which is split into a list - buffer_polygons = list(cell_buffers["geom_polygon"].unary_union) + buffer_polygons = list(cell_buffers["geom_polygon"].unary_union.geoms) # change the data type into geopandas geodataframe buffer_polygons_gdf = gpd.GeoDataFrame(geometry=buffer_polygons, crs=3035) @@ -422,6 +437,7 @@ def area_grouping( maximum_total_demand and "residential_and_service_demand" in join.columns ): + huge_areas_index = ( join.groupby("area_id").residential_and_service_demand.sum() > maximum_total_demand @@ -446,10 +462,19 @@ def area_grouping( join_2["area_id"] = join_2.index_right + max_area_id + 1 - join = join.append( - join_2[ - ["residential_and_service_demand", "geom_polygon", "area_id"] - ] + join = pd.concat( + [ + join, + join_2[ + [ + "zensus_population_id", + "residential_and_service_demand", + "geom_polygon", + "area_id", + ] + ], + ], + ignore_index=True, ) return join @@ -534,6 +559,14 @@ def district_heating_areas(scenario_name, plotting=False): district_heating_share = heat_parameters["DE_district_heating_share"] + minimum_connection_rate = 0.3 + + # Adjust minimum connection rate for status2019, and other statusquo scn + # otherwise the existing district heating grids would have too much demand + # if scenario_name == "status2019": + if "status" in scenario_name: + minimum_connection_rate = 0.6 + # heat_demand is scenario specific heat_demand_cells = load_heat_demands(scenario_name) @@ -542,7 +575,9 @@ def district_heating_areas(scenario_name, plotting=False): # by the area grouping function), load only the first returned result: [0] min_hd_census = 10000 / 3.6 # in MWh - census_plus_heat_demand = load_census_data()[0].copy() + census_plus_heat_demand = load_census_data( + minimum_connection_rate=minimum_connection_rate + )[0].copy() census_plus_heat_demand[ "residential_and_service_demand" ] = heat_demand_cells.loc[ @@ -574,7 +609,7 @@ def district_heating_areas(scenario_name, plotting=False): # ASSUMPTION HERE: 2035 HD defined the PSDs min_hd = 10000 / 3.6 PSDs = area_grouping( - select_high_heat_demands(load_heat_demands("eGon2035")), + select_high_heat_demands(load_heat_demands(scenario_name)), distance=200, minimum_total_demand=min_hd, ) @@ -612,15 +647,18 @@ def district_heating_areas(scenario_name, plotting=False): # group the resulting scenario specific district heating areas scenario_dh_area = area_grouping( - gpd.GeoDataFrame( - cells[["residential_and_service_demand", "geom_polygon"]].append( - new_areas[["residential_and_service_demand", "geom_polygon"]] - ), - geometry="geom_polygon", - ), + pd.concat( + [ + cells[["residential_and_service_demand", "geom_polygon"]], + new_areas[["residential_and_service_demand", "geom_polygon"]], + ] + ).reset_index(), distance=500, maximum_total_demand=4e6, ) + scenario_dh_area.loc[:, "zensus_population_id"] = scenario_dh_area.loc[ + :, "zensus_population_id" + ].astype(int) # scenario_dh_area.plot(column = "area_id") scenario_dh_area.groupby("area_id").size().sort_values() @@ -635,11 +673,12 @@ def district_heating_areas(scenario_name, plotting=False): f"""DELETE FROM demand.egon_map_zensus_district_heating_areas WHERE scenario = '{scenario_name}'""" ) - scenario_dh_area[["scenario", "area_id"]].to_sql( + scenario_dh_area[["scenario", "area_id", "zensus_population_id"]].to_sql( "egon_map_zensus_district_heating_areas", schema="demand", con=db.engine(), if_exists="append", + index=False, ) # Create polygons around the grouped cells and store them in the database @@ -675,7 +714,9 @@ def district_heating_areas(scenario_name, plotting=False): f"""DELETE FROM demand.egon_district_heating_areas WHERE scenario = '{scenario_name}'""" ) - areas_dissolved.reset_index().to_postgis( + areas_dissolved.reset_index().drop( + "zensus_population_id", axis="columns" + ).to_postgis( "egon_district_heating_areas", schema="demand", con=db.engine(), @@ -725,18 +766,7 @@ def add_metadata(): # Prepare variables license_district_heating_areas = [ - { - # this could be the license of the "district_heating_areas" - "name": "Creative Commons Attribution 4.0 International", - "title": "CC BY 4.0", - "path": "https://creativecommons.org/licenses/by/4.0/", - "instruction": ( - "You are free: To Share, To Adapt;" - " As long as you: Attribute!" - ), - "attribution": "© Europa-Universität Flensburg", # if all agree - # "attribution": "© ZNES Flensburg", # alternative - } + license_ccby("© Europa-Universität Flensburg") ] # Metadata creation for district heating areas (polygons) @@ -746,25 +776,14 @@ def add_metadata(): "description": "Modelled future district heating areas for " "the supply of residential and service-sector heat demands", "language": ["EN"], + "publicationDate": datetime.date.today().isoformat(), + "context": context(), "spatial": {"location": "", "extent": "Germany", "resolution": ""}, - "temporal": { - "referenceDate": "scenario-specific", - "timeseries": { - "start": "", - "end": "", - "resolution": "", - "alignment": "", - "aggregationType": "", - }, - }, "sources": [ - { - # eGon scenario specific heat demand distribution based - # on Peta5_0_1, using vg250 boundaries - }, - { - # Census gridded apartment data - }, + sources()["peta"], + sources()["egon-data"], + sources()["zensus"], + sources()["vg250"], ], "resources": [ { @@ -823,21 +842,21 @@ def add_metadata(): "licenses": license_district_heating_areas, "contributors": [ { - "title": "Eva, Clara", - "email": "", - "date": "2021-05-07", - "object": "", - "comment": "Processed data", - } - ], - "metaMetadata": { # https://github.com/OpenEnergyPlatform/oemetadata - "metadataVersion": "OEP-1.4.0", - "metadataLicense": { - "name": "CC0-1.0", - "title": "Creative Commons Zero v1.0 Universal", - "path": ("https://creativecommons.org/publicdomain/zero/1.0/"), + "title": "EvaWie", + "email": "http://github.com/EvaWie", + "date": time.strftime("%Y-%m-%d"), + "object": None, + "comment": "Imported data", + }, + { + "title": "Clara Büttner", + "email": "http://github.com/ClaraBuettner", + "date": time.strftime("%Y-%m-%d"), + "object": None, + "comment": "Updated metadata", }, - }, + ], + "metaMetadata": meta_metadata(), } meta_json = "'" + json.dumps(meta) + "'" @@ -851,25 +870,14 @@ def add_metadata(): " for supply of residential and service-sector heat demands" " assigned to zensus_population_ids", "language": ["EN"], + "publicationDate": datetime.date.today().isoformat(), + "context": context(), "spatial": {"location": "", "extent": "Germany", "resolution": ""}, - "temporal": { - "referenceDate": "scenario-specific", - "timeseries": { - "start": "", - "end": "", - "resolution": "", - "alignment": "", - "aggregationType": "", - }, - }, "sources": [ - { - # eGon scenario specific heat demand distribution based - # on Peta5_0_1, using vg250 boundaries - }, - { - # Census gridded apartment data - }, + sources()["peta"], + sources()["egon-data"], + sources()["zensus"], + sources()["vg250"], ], # Add the license for the map table "resources": [ @@ -924,21 +932,21 @@ def add_metadata(): "licenses": license_district_heating_areas, "contributors": [ { - "title": "Eva, Clara", - "email": "", - "date": "2021-05-07", - "object": "", - "comment": "Processed data", - } - ], - "metaMetadata": { # https://github.com/OpenEnergyPlatform/oemetadata - "metadataVersion": "OEP-1.4.0", - "metadataLicense": { - "name": "CC0-1.0", - "title": "Creative Commons Zero v1.0 Universal", - "path": ("https://creativecommons.org/publicdomain/zero/1.0/"), + "title": "EvaWie", + "email": "http://github.com/EvaWie", + "date": time.strftime("%Y-%m-%d"), + "object": None, + "comment": "Imported data", }, - }, + { + "title": "Clara Büttner", + "email": "http://github.com/ClaraBuettner", + "date": time.strftime("%Y-%m-%d"), + "object": None, + "comment": "Updated metadata", + }, + ], + "metaMetadata": meta_metadata(), } meta_json = "'" + json.dumps(meta) + "'" @@ -1158,12 +1166,11 @@ def demarcation(plotting=True): heat_density_per_scenario = {} # scenario specific district heating areas - heat_density_per_scenario["eGon2035"] = district_heating_areas( - "eGon2035", plotting - ) - heat_density_per_scenario["eGon100RE"] = district_heating_areas( - "eGon100RE", plotting - ) + + for scenario in config.settings()["egon-data"]["--scenarios"]: + heat_density_per_scenario[scenario] = district_heating_areas( + scenario, plotting + ) if plotting: plot_heat_density_sorted(heat_density_per_scenario) diff --git a/src/egon/data/datasets/district_heating_areas/plot.py b/src/egon/data/datasets/district_heating_areas/plot.py index ff1f83521..9cd3b2f58 100644 --- a/src/egon/data/datasets/district_heating_areas/plot.py +++ b/src/egon/data/datasets/district_heating_areas/plot.py @@ -53,13 +53,18 @@ def plot_heat_density_sorted(heat_denisty_per_scenario, scenario_name=None): fig, ax = plt.subplots(1, 1) colors = pd.DataFrame( - columns=["share", "curve"], index=["eGon2035", "eGon100RE"] + columns=["share", "curve"], index=["status2019", "status2023", "eGon2035", "eGon100RE"] ) colors["share"]["eGon2035"] = "darkblue" colors["curve"]["eGon2035"] = "blue" colors["share"]["eGon100RE"] = "red" colors["curve"]["eGon100RE"] = "orange" + colors["share"]["status2019"] = "darkgreen" + colors["curve"]["status2019"] = "green" + colors["share"]["status2023"] = "darkgrey" + colors["curve"]["status2023"] = "grey" + # TODO status2023 set plotting=False? for scenario in heat_denisty_per_scenario.keys(): diff --git a/src/egon/data/datasets/electrical_neighbours.py b/src/egon/data/datasets/electrical_neighbours.py old mode 100755 new mode 100644 index b60b198e5..952f6deb0 --- a/src/egon/data/datasets/electrical_neighbours.py +++ b/src/egon/data/datasets/electrical_neighbours.py @@ -1,19 +1,28 @@ """The central module containing all code dealing with electrical neighbours """ +from os import path +from pathlib import Path +import datetime +import logging +import os.path import zipfile -import geopandas as gpd -import pandas as pd from shapely.geometry import LineString from sqlalchemy.orm import sessionmaker +import entsoe +import geopandas as gpd +import pandas as pd +import requests -import egon.data.datasets.etrago_setup as etrago -import egon.data.datasets.scenario_parameters.parameters as scenario_parameters -from egon.data import config, db -from egon.data.datasets import Dataset +from egon.data import config, db, logger +from egon.data.db import session_scope +from egon.data.datasets import Dataset, wrapped_partial from egon.data.datasets.fill_etrago_gen import add_marginal_costs +from egon.data.datasets.fix_ehv_subnetworks import select_bus_id from egon.data.datasets.scenario_parameters import get_sector_parameters +import egon.data.datasets.etrago_setup as etrago +import egon.data.datasets.scenario_parameters.parameters as scenario_parameters class ElectricalNeighbours(Dataset): @@ -198,16 +207,22 @@ def buses(scenario, sources, targets): # if in test mode, add bus in center of Germany if config.settings()["egon-data"]["--dataset-boundary"] != "Everything": - central_buses = central_buses.append( - { - "scn_name": scenario, - "bus_id": next_bus_id, - "x": 10.4234469, - "y": 51.0834196, - "country": "DE", - "carrier": "AC", - "v_nom": 380.0, - }, + central_buses = pd.concat( + [ + central_buses, + pd.DataFrame( + index=[central_buses.index.max() + 1], + data={ + "scn_name": scenario, + "bus_id": next_bus_id, + "x": 10.4234469, + "y": 51.0834196, + "country": "DE", + "carrier": "AC", + "v_nom": 380.0, + }, + ), + ], ignore_index=True, ) next_bus_id += 1 @@ -220,38 +235,50 @@ def buses(scenario, sources, targets): for cntr in vnom_per_country.index: print(cntr) if 110.0 in vnom_per_country[cntr]: - central_buses = central_buses.append( - { - "scn_name": scenario, - "bus_id": next_bus_id, - "x": central_buses[ - central_buses.country == cntr - ].x.unique()[0], - "y": central_buses[ - central_buses.country == cntr - ].y.unique()[0], - "country": cntr, - "carrier": "AC", - "v_nom": 110.0, - }, + central_buses = pd.concat( + [ + central_buses, + pd.DataFrame( + index=[central_buses.index.max() + 1], + data={ + "scn_name": scenario, + "bus_id": next_bus_id, + "x": central_buses[ + central_buses.country == cntr + ].x.unique()[0], + "y": central_buses[ + central_buses.country == cntr + ].y.unique()[0], + "country": cntr, + "carrier": "AC", + "v_nom": 110.0, + }, + ), + ], ignore_index=True, ) next_bus_id += 1 if 220.0 in vnom_per_country[cntr]: - central_buses = central_buses.append( - { - "scn_name": scenario, - "bus_id": next_bus_id, - "x": central_buses[ - central_buses.country == cntr - ].x.unique()[0], - "y": central_buses[ - central_buses.country == cntr - ].y.unique()[0], - "country": cntr, - "carrier": "AC", - "v_nom": 220.0, - }, + central_buses = pd.concat( + [ + central_buses, + pd.DataFrame( + index=[central_buses.index.max() + 1], + data={ + "scn_name": scenario, + "bus_id": next_bus_id, + "x": central_buses[ + central_buses.country == cntr + ].x.unique()[0], + "y": central_buses[ + central_buses.country == cntr + ].y.unique()[0], + "country": cntr, + "carrier": "AC", + "v_nom": 220.0, + }, + ), + ], ignore_index=True, ) next_bus_id += 1 @@ -269,7 +296,11 @@ def buses(scenario, sources, targets): central_buses.scn_name = scenario # Insert all central buses for eGon2035 - if scenario == "eGon2035": + if scenario in [ + "eGon2035", + "status2019", + "status2023", + ]: # TODO: status2023 this is hardcoded shit central_buses.to_postgis( targets["buses"]["table"], schema=targets["buses"]["schema"], @@ -354,10 +385,11 @@ def cross_border_lines(scenario, sources, targets, central_buses): lines.loc[:, "foreign_bus"] = lines.loc[:, "foreign_bus"].astype(int) # Copy all parameters from border-crossing lines - new_lines = lines.copy() + new_lines = lines.copy().set_crs(4326) # Set bus0 as foreign_bus from osmtgmod new_lines.bus0 = new_lines.foreign_bus.copy() + new_lines.bus0 = new_lines.bus0.astype(int) # Add country tag and set index new_lines["country"] = ( @@ -392,8 +424,7 @@ def cross_border_lines(scenario, sources, targets, central_buses): ) # Set topo as geometry column - new_lines = new_lines.set_geometry("topo") - + new_lines = new_lines.set_geometry("topo").set_crs(4326) # Calcultae length of lines based on topology old_length = new_lines["length"].copy() new_lines["length"] = new_lines.to_crs(3035).length / 1000 @@ -534,17 +565,23 @@ def central_transformer(scenario, sources, targets, central_buses, new_lines): s_nom, x = choose_transformer(s_nom) - trafo = trafo.append( - { - "trafo_id": trafo_id, - "bus0": row.bus_id, - "bus1": central_buses[ - (central_buses.v_nom == 380) - & (central_buses.country == row.country) - ].bus_id.values[0], - "s_nom": s_nom, - "x": x, - }, + trafo = pd.concat( + [ + trafo, + pd.DataFrame( + index=[trafo.index.max() + 1], + data={ + "trafo_id": trafo_id, + "bus0": row.bus_id, + "bus1": central_buses[ + (central_buses.v_nom == 380) + & (central_buses.country == row.country) + ].bus_id.values[0], + "s_nom": s_nom, + "x": x, + }, + ), + ], ignore_index=True, ) trafo_id += 1 @@ -603,23 +640,19 @@ def foreign_dc_lines(scenario, sources, targets, central_buses): AND country != 'DE') """ ) - capital_cost = get_sector_parameters("electricity", "eGon2035")[ + capital_cost = get_sector_parameters("electricity", scenario)[ "capital_cost" ] # Add DC line from Lübeck to Sweden - converter_luebeck = db.select_dataframe( - f""" - SELECT bus_id FROM - {sources['electricity_buses']['schema']}. - {sources['electricity_buses']['table']} - WHERE x = 10.802358024202768 - AND y = 53.897547401787 - AND v_nom = 380 - AND scn_name = '{scenario}' - AND carrier = 'AC' - """ - ).squeeze() + converter_luebeck = select_bus_id( + 10.802358024202768, + 53.897547401787, + 380, + scenario, + "AC", + find_closest=True, + ) foreign_links = pd.DataFrame( index=[0], @@ -638,36 +671,35 @@ def foreign_dc_lines(scenario, sources, targets, central_buses): # When not in test-mode, add DC line from Bentwisch to Denmark if config.settings()["egon-data"]["--dataset-boundary"] == "Everything": - converter_bentwisch = db.select_dataframe( - f""" - SELECT bus_id FROM - {sources['electricity_buses']['schema']}. - {sources['electricity_buses']['table']} - WHERE x = 12.213671694775988 - AND y = 54.09974494662279 - AND v_nom = 380 - AND scn_name = '{scenario}' - AND carrier = 'AC' - """ - ).squeeze() - - foreign_links = foreign_links.append( - pd.DataFrame( - index=[1], - data={ - "link_id": db.next_etrago_id("link") + 1, - "bus0": converter_bentwisch, - "bus1": central_buses[ - (central_buses.country == "DK") - & (central_buses.v_nom == 380) - & (central_buses.x > 10) - ] - .squeeze() - .bus_id, - "p_nom": 600, - "length": 170, - }, - ) + converter_bentwisch = select_bus_id( + 12.213671694775988, + 54.09974494662279, + 380, + scenario, + "AC", + find_closest=True, + ) + + foreign_links = pd.concat( + [ + foreign_links, + pd.DataFrame( + index=[1], + data={ + "link_id": db.next_etrago_id("link") + 1, + "bus0": converter_bentwisch, + "bus1": central_buses[ + (central_buses.country == "DK") + & (central_buses.v_nom == 380) + & (central_buses.x > 10) + ] + .squeeze() + .bus_id, + "p_nom": 600, + "length": 170, + }, + ), + ] ) # Set parameters for all DC lines @@ -707,7 +739,7 @@ def grid(): sources = config.datasets()["electrical_neighbours"]["sources"] targets = config.datasets()["electrical_neighbours"]["targets"] - for scenario in ["eGon2035"]: + for scenario in config.settings()["egon-data"]["--scenarios"]: central_buses = buses(scenario, sources, targets) foreign_lines = cross_border_lines( @@ -771,7 +803,7 @@ def map_carriers_tyndp(): } -def get_foreign_bus_id(): +def get_foreign_bus_id(scenario): """Calculte the etrago bus id from Nodes of TYNDP based on the geometry Returns @@ -784,9 +816,9 @@ def get_foreign_bus_id(): sources = config.datasets()["electrical_neighbours"]["sources"] bus_id = db.select_geodataframe( - """SELECT bus_id, ST_Buffer(geom, 1) as geom, country + f"""SELECT bus_id, ST_Buffer(geom, 1) as geom, country FROM grid.egon_etrago_bus - WHERE scn_name = 'eGon2035' + WHERE scn_name = '{scenario}' AND carrier = 'AC' AND v_nom = 380. AND country != 'DE' @@ -899,7 +931,7 @@ def calc_capacities(): ] -def insert_generators(capacities): +def insert_generators_tyndp(capacities): """Insert generators for foreign countries based on TYNDP-data Parameters @@ -971,7 +1003,9 @@ def insert_generators(capacities): ) gen.loc[:, "bus"] = ( - get_foreign_bus_id().loc[gen.loc[:, "Node/Line"]].values + get_foreign_bus_id(scenario="eGon2035") + .loc[gen.loc[:, "Node/Line"]] + .values ) # Add scenario column @@ -1055,7 +1089,7 @@ def insert_generators(capacities): session.commit() -def insert_storage(capacities): +def insert_storage_tyndp(capacities): """Insert storage units for foreign countries based on TYNDP-data Parameters @@ -1107,7 +1141,9 @@ def insert_storage(capacities): ) store.loc[:, "bus"] = ( - get_foreign_bus_id().loc[store.loc[:, "Node/Line"]].values + get_foreign_bus_id(scenario="eGon2035") + .loc[store.loc[:, "Node/Line"]] + .values ) # Add columns for additional parameters to df @@ -1124,9 +1160,9 @@ def insert_storage(capacities): for x in parameters: store.loc[store["carrier"] == "battery", x] = parameters_battery[x] - store.loc[ - store["carrier"] == "pumped_hydro", x - ] = parameters_pumped_hydro[x] + store.loc[store["carrier"] == "pumped_hydro", x] = ( + parameters_pumped_hydro[x] + ) # insert data session = sessionmaker(bind=db.engine())() @@ -1184,9 +1220,9 @@ def tyndp_generation(): capacities = calc_capacities() - insert_generators(capacities) + insert_generators_tyndp(capacities) - insert_storage(capacities) + insert_storage_tyndp(capacities) def tyndp_demand(): @@ -1247,11 +1283,13 @@ def tyndp_demand(): ] # Assign etrago bus_id to TYNDP nodes buses = pd.DataFrame({"nodes": nodes}) - buses.loc[ - buses[buses.nodes.isin(map_buses.keys())].index, "nodes" - ] = buses[buses.nodes.isin(map_buses.keys())].nodes.map(map_buses) + buses.loc[buses[buses.nodes.isin(map_buses.keys())].index, "nodes"] = ( + buses[buses.nodes.isin(map_buses.keys())].nodes.map(map_buses) + ) buses.loc[:, "bus"] = ( - get_foreign_bus_id().loc[buses.loc[:, "nodes"]].values + get_foreign_bus_id(scenario="eGon2035") + .loc[buses.loc[:, "nodes"]] + .values ) buses.set_index("nodes", inplace=True) buses = buses[~buses.index.duplicated(keep="first")] @@ -1312,3 +1350,696 @@ def tyndp_demand(): session.add(entry) session.add(entry_ts) session.commit() + + +def get_entsoe_token(): + """Check for token in home dir. If not exists, check in working dir""" + token_path = path.join(path.expanduser("~"), ".entsoe-token") + if not os.path.isfile(token_path): + logger.info( + f"Token file not found at {token_path}. Will check in working directory." + ) + token_path = Path(".entsoe-token") + if os.path.isfile(token_path): + logger.info(f"Token found at {token_path}") + entsoe_token = open(token_path, "r").read(36) + if entsoe_token is None: + raise FileNotFoundError("No entsoe-token found.") + return entsoe_token + + +def entsoe_historic_generation_capacities( + year_start="20190101", year_end="20200101" +): + entsoe_token = get_entsoe_token() + client = entsoe.EntsoePandasClient(api_key=entsoe_token) + + start = pd.Timestamp(year_start, tz="Europe/Brussels") + end = pd.Timestamp(year_end, tz="Europe/Brussels") + start_gb = pd.Timestamp(year_start, tz="Europe/London") + end_gb = pd.Timestamp(year_end, tz="Europe/London") + countries = [ + "LU", + "AT", + "FR", + "NL", + "CZ", + "DK_1", + "DK_2", + "PL", + "CH", + "NO", + "BE", + "SE", + "GB", + ] + # No GB data after Brexit + if int(year_start[:4]) > 2021: + logger.warning( + "No GB data after Brexit. GB is dropped from entsoe query!" + ) + countries = [c for c in countries if c != "GB"] + # todo: define wanted countries + + not_retrieved = [] + dfs = [] + for country in countries: + if country == "GB": + kwargs = dict(start=start_gb, end=end_gb) + else: + kwargs = dict(start=start, end=end) + try: + country_data = client.query_installed_generation_capacity( + country, **kwargs + ) + dfs.append(country_data) + except (entsoe.exceptions.NoMatchingDataError, requests.HTTPError): + logger.warning( + f"Data for country: {country} could not be retrieved." + ) + not_retrieved.append(country) + pass + + if dfs: + df = pd.concat(dfs) + df["country"] = [c for c in countries if c not in not_retrieved] + df.set_index("country", inplace=True) + if int(year_start[:4]) == 2023: + # https://www.bmreports.com/bmrs/?q=foregeneration/capacityaggregated + # could probably somehow be automised + # https://www.elexonportal.co.uk/category/view/178 + # in MW + installed_capacity_gb = pd.Series( + { + "Biomass": 4438, + "Fossil Gas": 37047, + "Fossil Hard coal": 1491, + "Hydro Pumped Storage": 5603, + "Hydro Run-of-river and poundage": 2063, + "Nuclear": 4950, + "Other": 3313, + "Other renewable": 1462, + "Solar": 14518, + "Wind Offshore": 13038, + "Wind Onshore": 13907, + }, + name="GB", + ) + df = pd.concat([df.T, installed_capacity_gb], axis=1).T + logger.info("Manually added generation capacities for GB 2023.") + not_retrieved = [c for c in not_retrieved if c != "GB"] + df.fillna(0, inplace=True) + else: + df = pd.DataFrame() + return df, not_retrieved + + +def entsoe_historic_demand(year_start="20190101", year_end="20200101"): + entsoe_token = get_entsoe_token() + client = entsoe.EntsoePandasClient(api_key=entsoe_token) + + start = pd.Timestamp(year_start, tz="Europe/Brussels") + end = pd.Timestamp(year_end, tz="Europe/Brussels") + start_gb = start.tz_convert("Europe/London") + end_gb = end.tz_convert("Europe/London") + + countries = [ + "LU", + "AT", + "FR", + "NL", + "CZ", + "DK_1", + "DK_2", + "PL", + "CH", + "NO", + "BE", + "SE", + "GB", + ] + + # todo: define wanted countries + + not_retrieved = [] + dfs = [] + + for country in countries: + if country == "GB": + kwargs = dict(start=start_gb, end=end_gb) + else: + kwargs = dict(start=start, end=end) + try: + country_data = ( + client.query_load(country, **kwargs) + .resample("H")["Actual Load"] + .mean() + ) + if country == "GB": + country_data.index = country_data.index.tz_convert( + "Europe/Brussels" + ) + dfs.append(country_data) + except (entsoe.exceptions.NoMatchingDataError, requests.HTTPError): + not_retrieved.append(country) + logger.warning( + f"Data for country: {country} could not be retrieved." + ) + pass + + if dfs: + df = pd.concat(dfs, axis=1) + df.columns = [c for c in countries if c not in not_retrieved] + df.index = pd.date_range(year_start, periods=8760, freq="H") + else: + df = pd.DataFrame() + return df, not_retrieved + + +def map_carriers_entsoe(): + """Map carriers from entsoe-data to carriers used in eGon + Returns + ------- + dict + Carrier from entsoe to eGon + """ + return { + "Biomass": "biomass", + "Fossil Brown coal/Lignite": "lignite", + "Fossil Coal-derived gas": "coal", + "Fossil Gas": "OCGT", + "Fossil Hard coal": "coal", + "Fossil Oil": "oil", + "Fossil Oil shale": "oil", + "Fossil Peat": "others", + "Geothermal": "geo_thermal", + "Hydro Pumped Storage": "Hydro Pumped Storage", + "Hydro Run-of-river and poundage": "run_of_river", + "Hydro Water Reservoir": "reservoir", + "Marine": "others", + "Nuclear": "nuclear", + "Other": "others", + "Other renewable": "others", + "Solar": "solar", + "Waste": "others", + "Wind Offshore": "wind_offshore", + "Wind Onshore": "wind_onshore", + } + + +def entsoe_to_bus_etrago(scn_name): + map_entsoe = pd.Series( + { + "LU": "LU00", + "AT": "AT00", + "FR": "FR00", + "NL": "NL00", + "DK_1": "DK00", + "DK_2": "DKE1", + "PL": "PL00", + "CH": "CH00", + "NO": "NO00", + "BE": "BE00", + "SE": "SE00", + "GB": "UK00", + "CZ": "CZ00", + } + ) + + for_bus = get_foreign_bus_id(scenario=scn_name) + + return map_entsoe.map(for_bus) + + +def save_entsoe_data(df: pd.DataFrame, file_path: Path): + os.makedirs(file_path.parent, exist_ok=True) + if not df.empty: + df.to_csv(file_path, index_label="Index") + logger.info( + f"Saved entsoe data for {file_path.stem} " + f"to {file_path.parent} for countries: {df.index}" + ) + + +def fill_by_backup_data_from_former_runs(df_sq, file_path, not_retrieved): + """ + Fills missing data from former runs + Parameters + ---------- + df_sq: pd.DataFrame + file_path: str, Path + not_retrieved: list + + Returns + ------- + df_sq, not_retrieved + + """ + sq_backup = pd.read_csv(file_path, index_col="Index") + # check for missing columns in backup (former runs) + c_backup = [c for c in sq_backup.columns if c in not_retrieved] + # remove columns, if found in backup + not_retrieved = [c for c in not_retrieved if c not in c_backup] + if c_backup: + df_sq = pd.concat([df_sq, sq_backup.loc[:, c_backup]], axis=1) + logger.info(f"Appended data from former runs for {c_backup}") + return df_sq, not_retrieved + + +def insert_generators_sq(scn_name="status2019"): + """ + Insert generators for foreign countries based on ENTSO-E data + + Parameters + ---------- + gen_sq : pandas dataframe + df with all the foreign generators produced by the function + entsoe_historic_generation_capacities + scn_name : str + The default is "status2019". + + Returns + ------- + None. + + """ + if "status" in scn_name: + year = int(scn_name.split("status")[-1]) + year_start_end = { + "year_start": f"{year}0101", + "year_end": f"{year+1}0101", + } + else: + raise ValueError("No valid scenario name!") + + df_gen_sq, not_retrieved = entsoe_historic_generation_capacities( + **year_start_end + ) + + if not_retrieved: + logger.warning("Generation data from entsoe could not be retrieved.") + # check for generation backup from former runs + file_path = Path( + "./", "entsoe_data", f"gen_entsoe_{scn_name}.csv" + ).resolve() + if os.path.isfile(file_path): + df_gen_sq, not_retrieved = fill_by_backup_data_from_former_runs( + df_gen_sq, file_path, not_retrieved + ) + save_entsoe_data(df_gen_sq, file_path=file_path) + + if not_retrieved: + logger.warning( + f"Backup data of 2019 is used instead for {not_retrieved}" + ) + df_gen_sq_backup = pd.read_csv( + "data_bundle_egon_data/entsoe/gen_entsoe.csv", index_col="Index" + ) + df_gen_sq = pd.concat( + [df_gen_sq, df_gen_sq_backup.loc[not_retrieved]], axis=1 + ) + + targets = config.datasets()["electrical_neighbours"]["targets"] + # Delete existing data + db.execute_sql( + f""" + DELETE FROM + {targets['generators']['schema']}.{targets['generators']['table']} + WHERE bus IN ( + SELECT bus_id FROM + {targets['buses']['schema']}.{targets['buses']['table']} + WHERE country != 'DE' + AND scn_name = '{scn_name}') + AND scn_name = '{scn_name}' + AND carrier != 'CH4' + """ + ) + + db.execute_sql( + f""" + DELETE FROM + {targets['generators_timeseries']['schema']}. + {targets['generators_timeseries']['table']} + WHERE generator_id NOT IN ( + SELECT generator_id FROM + {targets['generators']['schema']}.{targets['generators']['table']} + ) + AND scn_name = '{scn_name}' + """ + ) + entsoe_to_bus = entsoe_to_bus_etrago(scn_name) + carrier_entsoe = map_carriers_entsoe() + df_gen_sq = df_gen_sq.groupby(axis=1, by=carrier_entsoe).sum() + + # Filter generators modeled as storage and geothermal + df_gen_sq = df_gen_sq.loc[ + :, ~df_gen_sq.columns.isin(["Hydro Pumped Storage", "geo_thermal"]) + ] + + list_gen_sq = pd.DataFrame( + dtype=int, columns=["carrier", "country", "capacity"] + ) + for carrier in df_gen_sq.columns: + gen_carry = df_gen_sq[carrier] + for country, cap in gen_carry.items(): + gen = pd.DataFrame( + {"carrier": carrier, "country": country, "capacity": cap}, + index=[1], + ) + # print(gen) + list_gen_sq = pd.concat([list_gen_sq, gen], ignore_index=True) + + list_gen_sq = list_gen_sq[list_gen_sq.capacity > 0] + list_gen_sq["scenario"] = scn_name + + # Add marginal costs + list_gen_sq = add_marginal_costs(list_gen_sq) + + # Find foreign bus to assign the generator + list_gen_sq["bus"] = list_gen_sq.country.map(entsoe_to_bus) + + # insert generators data + session = sessionmaker(bind=db.engine())() + for i, row in list_gen_sq.iterrows(): + entry = etrago.EgonPfHvGenerator( + scn_name=row.scenario, + generator_id=int(db.next_etrago_id("generator")), + bus=row.bus, + carrier=row.carrier, + p_nom=row.capacity, + marginal_cost=row.marginal_cost, + ) + + session.add(entry) + session.commit() + + # assign generators time-series data + renew_carriers_sq = ["wind_onshore", "wind_offshore", "solar"] + + sql = f"""SELECT * FROM + {targets['generators_timeseries']['schema']}. + {targets['generators_timeseries']['table']} + WHERE scn_name = 'eGon100RE' + """ + series_egon100 = pd.read_sql_query(sql, db.engine()) + + sql = f""" SELECT * FROM + {targets['generators']['schema']}.{targets['generators']['table']} + WHERE bus IN ( + SELECT bus_id FROM + {targets['buses']['schema']}.{targets['buses']['table']} + WHERE country != 'DE' + AND scn_name = '{scn_name}') + AND scn_name = '{scn_name}' + """ + df_gen_sq = pd.read_sql_query(sql, db.engine()) + df_gen_sq = df_gen_sq[df_gen_sq.carrier.isin(renew_carriers_sq)] + + sql = f""" SELECT * FROM + {targets['generators']['schema']}.{targets['generators']['table']} + WHERE bus IN ( + SELECT bus_id FROM + {targets['buses']['schema']}.{targets['buses']['table']} + WHERE country != 'DE' + AND scn_name = 'eGon100RE') + AND scn_name = 'eGon100RE' + """ + gen_100 = pd.read_sql_query(sql, db.engine()) + gen_100 = gen_100[gen_100["carrier"].isin(renew_carriers_sq)] + + # egon_sq_to_100 map the timeseries used in the scenario eGon100RE + # to the same bus and carrier for the status quo scenario + egon_sq_to_100 = {} + for i, gen in df_gen_sq.iterrows(): + gen_id_100 = gen_100[ + (gen_100["bus"] == gen["bus"]) + & (gen_100["carrier"] == gen["carrier"]) + ]["generator_id"].values[0] + + egon_sq_to_100[gen["generator_id"]] = gen_id_100 + + # insert generators_timeseries data + with session_scope() as session: + for gen_id in df_gen_sq.generator_id: + serie = series_egon100[ + series_egon100.generator_id == egon_sq_to_100[gen_id] + ]["p_max_pu"].values[0] + entry = etrago.EgonPfHvGeneratorTimeseries( + scn_name=scn_name, generator_id=gen_id, temp_id=1, p_max_pu=serie + ) + + session.add(entry) + session.commit() + + return + + +def insert_storage_units_sq(scn_name="status2019"): + """ + Insert storage_units for foreign countries based on ENTSO-E data + + Parameters + ---------- + scn_name : str + Scenario to which the foreign storage units will be assigned. + The default is "status2019". + + Returns + ------- + None. + + """ + if "status" in scn_name: + year = int(scn_name.split("status")[-1]) + year_start_end = { + "year_start": f"{year}0101", + "year_end": f"{year+1}0101", + } + else: + raise ValueError("No valid scenario name!") + + df_gen_sq, not_retrieved = entsoe_historic_generation_capacities( + **year_start_end + ) + + if not_retrieved: + logger.warning("Generation data from entsoe could not be retrieved.") + # check for generation backup from former runs + file_path = Path( + "./", "entsoe_data", f"gen_entsoe_{scn_name}.csv" + ).resolve() + if os.path.isfile(file_path): + df_gen_sq, not_retrieved = fill_by_backup_data_from_former_runs( + df_gen_sq, file_path, not_retrieved + ) + save_entsoe_data(df_gen_sq, file_path=file_path) + + if not_retrieved: + logger.warning( + f"Backup data of 2019 is used instead for {not_retrieved}" + ) + df_gen_sq_backup = pd.read_csv( + "data_bundle_egon_data/entsoe/gen_entsoe.csv", index_col="Index" + ) + df_gen_sq = pd.concat( + [df_gen_sq, df_gen_sq_backup.loc[not_retrieved]], axis=1 + ) + + sto_sq = df_gen_sq.loc[:, df_gen_sq.columns == "Hydro Pumped Storage"] + sto_sq.rename(columns={"Hydro Pumped Storage": "p_nom"}, inplace=True) + + targets = config.datasets()["electrical_neighbours"]["targets"] + + # Delete existing data + db.execute_sql( + f""" + DELETE FROM {targets['storage']['schema']}.{targets['storage']['table']} + WHERE bus IN ( + SELECT bus_id FROM + {targets['buses']['schema']}.{targets['buses']['table']} + WHERE country != 'DE' + AND scn_name = '{scn_name}') + AND scn_name = '{scn_name}' + """ + ) + + # Add missing information suitable for eTraGo selected from scenario_parameter table + parameters_pumped_hydro = get_sector_parameters(sector="electricity", scenario=scn_name)["efficiency"]["pumped_hydro"] + + # Set bus_id + entsoe_to_bus = entsoe_to_bus_etrago(scn_name=scn_name) + sto_sq["bus"] = sto_sq.index.map(entsoe_to_bus) + + # Insert carrier specific parameters + sto_sq["carrier"] = "pumped_hydro" + sto_sq["scn_name"] = scn_name + sto_sq["dispatch"] = parameters_pumped_hydro["dispatch"] + sto_sq["store"] = parameters_pumped_hydro["store"] + sto_sq["standing_loss"] = parameters_pumped_hydro["standing_loss"] + sto_sq["max_hours"] = parameters_pumped_hydro["max_hours"] + + # Delete entrances without any installed capacity + sto_sq = sto_sq[sto_sq["p_nom"] > 0] + + # insert data pumped_hydro storage + + with session_scope() as session: + for i, row in sto_sq.iterrows(): + entry = etrago.EgonPfHvStorage( + scn_name=scn_name, + storage_id=int(db.next_etrago_id("storage")), + bus=row.bus, + max_hours=row.max_hours, + efficiency_store=row.store, + efficiency_dispatch=row.dispatch, + standing_loss=row.standing_loss, + carrier=row.carrier, + p_nom=row.p_nom, + ) + + session.add(entry) + session.commit() + + +def insert_loads_sq(scn_name="status2019"): + """ + Copy load timeseries data from entso-e. + + Returns + ------- + None. + + """ + sources = config.datasets()["electrical_neighbours"]["sources"] + targets = config.datasets()["electrical_neighbours"]["targets"] + + if scn_name == "status2019": + year_start_end = {"year_start": "20190101", "year_end": "20200101"} + elif scn_name == "status2023": + year_start_end = {"year_start": "20230101", "year_end": "20240101"} + else: + raise ValueError("No valid scenario name!") + + df_load_sq, not_retrieved = entsoe_historic_demand(**year_start_end) + + if not_retrieved: + logger.warning("Demand data from entsoe could not be retrieved.") + # check for generation backup from former runs + file_path = Path( + "./", "entsoe_data", f"load_entsoe_{scn_name}.csv" + ).resolve() + if os.path.isfile(file_path): + df_load_sq, not_retrieved = fill_by_backup_data_from_former_runs( + df_load_sq, file_path, not_retrieved + ) + save_entsoe_data(df_load_sq, file_path=file_path) + + if not_retrieved: + logger.warning( + f"Backup data of 2019 is used instead for {not_retrieved}" + ) + df_load_sq_backup = pd.read_csv( + "data_bundle_egon_data/entsoe/load_entsoe.csv", index_col="Index" + ) + df_load_sq_backup.index = df_load_sq.index + df_load_sq = pd.concat( + [df_load_sq, df_load_sq_backup.loc[:, not_retrieved]], axis=1 + ) + + # Delete existing data + db.execute_sql( + f""" + DELETE FROM {targets['load_timeseries']['schema']}. + {targets['load_timeseries']['table']} + WHERE + scn_name = '{scn_name}' + AND load_id IN ( + SELECT load_id FROM {targets['loads']['schema']}. + {targets['loads']['table']} + WHERE + scn_name = '{scn_name}' + AND carrier = 'AC' + AND bus NOT IN ( + SELECT bus_i + FROM {sources['osmtgmod_bus']['schema']}. + {sources['osmtgmod_bus']['table']})) + """ + ) + + db.execute_sql( + f""" + DELETE FROM {targets['loads']['schema']}. + {targets['loads']['table']} + WHERE + scn_name = '{scn_name}' + AND carrier = 'AC' + AND bus NOT IN ( + SELECT bus_i + FROM {sources['osmtgmod_bus']['schema']}. + {sources['osmtgmod_bus']['table']}) + """ + ) + + # get the corresponding bus per foreign country + entsoe_to_bus = entsoe_to_bus_etrago(scn_name) + + # Calculate and insert demand timeseries per etrago bus_id + with session_scope() as session: + for country in df_load_sq.columns: + load_id = db.next_etrago_id("load") + + entry = etrago.EgonPfHvLoad( + scn_name=scn_name, + load_id=int(load_id), + carrier="AC", + bus=int(entsoe_to_bus[country]), + ) + + entry_ts = etrago.EgonPfHvLoadTimeseries( + scn_name=scn_name, + load_id=int(load_id), + temp_id=1, + p_set=list(df_load_sq[country]), + ) + + session.add(entry) + session.add(entry_ts) + session.commit() + + +tasks = (grid,) + +insert_per_scenario = set() + +for scn_name in config.settings()["egon-data"]["--scenarios"]: + + if scn_name == "eGon2035": + insert_per_scenario.update([tyndp_generation, tyndp_demand]) + + if "status" in scn_name: + postfix = f"_{scn_name.split('status')[-1]}" + insert_per_scenario.update( + [ + wrapped_partial( + insert_generators_sq, scn_name=scn_name, postfix=postfix + ), + wrapped_partial( + insert_loads_sq, scn_name=scn_name, postfix=postfix + ), + wrapped_partial( + insert_storage_units_sq, scn_name=scn_name, postfix=postfix + ), + ] + ) + +tasks = tasks + (insert_per_scenario,) + + +class ElectricalNeighbours(Dataset): + def __init__(self, dependencies): + super().__init__( + name="ElectricalNeighbours", + version="0.0.11", + dependencies=dependencies, + tasks=tasks, + ) diff --git a/src/egon/data/datasets/electricity_demand/__init__.py b/src/egon/data/datasets/electricity_demand/__init__.py index 067ca90f6..8ebdce98a 100644 --- a/src/egon/data/datasets/electricity_demand/__init__.py +++ b/src/egon/data/datasets/electricity_demand/__init__.py @@ -28,7 +28,7 @@ class HouseholdElectricityDemand(Dataset): def __init__(self, dependencies): super().__init__( name="HouseholdElectricityDemand", - version="0.0.3", + version="0.0.5", dependencies=dependencies, tasks=(create_tables, get_annual_household_el_demand_cells), ) @@ -92,6 +92,8 @@ def get_annual_household_el_demand_cells(): session.query( HouseholdElectricityProfilesOfBuildings, HouseholdElectricityProfilesInCensusCells.nuts3, + HouseholdElectricityProfilesInCensusCells.factor_2019, + HouseholdElectricityProfilesInCensusCells.factor_2023, HouseholdElectricityProfilesInCensusCells.factor_2035, HouseholdElectricityProfilesInCensusCells.factor_2050, ) @@ -113,6 +115,8 @@ def ve(s): raise (ValueError(s)) dataset = egon.data.config.settings()["egon-data"]["--dataset-boundary"] + scenarios = egon.data.config.settings()["egon-data"]["--scenarios"] + iterate_over = ( "nuts3" if dataset == "Everything" @@ -122,23 +126,37 @@ def ve(s): ) df_annual_demand = pd.DataFrame( - columns=["eGon2035", "eGon100RE", "zensus_population_id"] + columns=scenarios + ["zensus_population_id"] ) for _, df in df_buildings_and_profiles.groupby(by=iterate_over): df_annual_demand_iter = pd.DataFrame( - columns=["eGon2035", "eGon100RE", "zensus_population_id"] - ) - df_annual_demand_iter["eGon2035"] = ( - df_profiles.loc[:, df["profile_id"]].sum(axis=0) - * df["factor_2035"].values - ) - df_annual_demand_iter["eGon100RE"] = ( - df_profiles.loc[:, df["profile_id"]].sum(axis=0) - * df["factor_2050"].values + columns=scenarios + ["zensus_population_id"] ) + + if "eGon2035" in scenarios: + df_annual_demand_iter["eGon2035"] = ( + df_profiles.loc[:, df["profile_id"]].sum(axis=0) + * df["factor_2035"].values + ) + if "eGon100RE" in scenarios: + df_annual_demand_iter["eGon100RE"] = ( + df_profiles.loc[:, df["profile_id"]].sum(axis=0) + * df["factor_2050"].values + ) + if "status2019" in scenarios: + df_annual_demand_iter["status2019"] = ( + df_profiles.loc[:, df["profile_id"]].sum(axis=0) + * df["factor_2019"].values + ) + + if "status2023" in scenarios: + df_annual_demand_iter["status2023"] = ( + df_profiles.loc[:, df["profile_id"]].sum(axis=0) + * df["factor_2023"].values + ) df_annual_demand_iter["zensus_population_id"] = df["cell_id"].values - df_annual_demand = df_annual_demand.append(df_annual_demand_iter) + df_annual_demand = pd.concat([df_annual_demand, df_annual_demand_iter]) df_annual_demand = ( df_annual_demand.groupby("zensus_population_id").sum().reset_index() @@ -200,8 +218,7 @@ def distribute_cts_demands(): ) # Insert data per scenario - for scn in sources["demandregio"]["scenarios"]: - + for scn in egon.data.config.settings()["egon-data"]["--scenarios"]: # Select heat_demand per zensus cell peta = db.select_dataframe( f"""SELECT zensus_population_id, demand as heat_demand, @@ -217,9 +234,10 @@ def distribute_cts_demands(): peta["nuts3"] = map_nuts3.nuts3 # Calculate share of nuts3 heat demand per zensus cell - peta["share"] = peta.heat_demand.groupby(peta.nuts3).apply( - lambda grp: grp / grp.sum() - ) + for nuts3, df in peta.groupby("nuts3"): + peta.loc[df.index, "share"] = ( + df["heat_demand"] / df["heat_demand"].sum() + ) # Select forecasted electrical demands from demandregio table demand_nuts3 = db.select_dataframe( diff --git a/src/egon/data/datasets/electricity_demand/temporal.py b/src/egon/data/datasets/electricity_demand/temporal.py index 73f5f62df..6e59e9607 100644 --- a/src/egon/data/datasets/electricity_demand/temporal.py +++ b/src/egon/data/datasets/electricity_demand/temporal.py @@ -10,6 +10,8 @@ from sqlalchemy import ARRAY, Column, Float, Integer, String from sqlalchemy.ext.declarative import declarative_base +import egon.data.datasets.scenario_parameters.parameters as scenario_parameters + Base = declarative_base() @@ -35,13 +37,15 @@ def create_table(): EgonEtragoElectricityCts.__table__.create(bind=engine, checkfirst=True) -def calc_load_curve(share_wz, annual_demand=1): - """ Create aggregated demand curve for service sector +def calc_load_curve(share_wz, scn, annual_demand=1): + """Create aggregated demand curve for service sector Parameters ---------- share_wz : pandas.Series or pandas.DataFrame Share of annual demand per cts branch + scn : str + Scenario name annual_demand : float or pandas.Series, optional Annual demand in MWh. The default is 1. @@ -51,7 +55,7 @@ def calc_load_curve(share_wz, annual_demand=1): Annual load curve of combindes cts branches """ - year = 2011 + year = int(scenario_parameters.global_settings(scn)["weather_year"]) sources = egon.data.config.datasets()["electrical_load_curves_cts"][ "sources" @@ -72,7 +76,7 @@ def calc_load_curve(share_wz, annual_demand=1): start=f"01/01/{year}", end=f"01/01/{year+1}", freq="H", - closed="left", + inclusive="left", ) ) @@ -86,7 +90,6 @@ def calc_load_curve(share_wz, annual_demand=1): # If shares per cts branch is a DataFrame (e.g. shares per substation) # demand curves are created for each row if type(share_wz) == pd.core.frame.DataFrame: - # Replace NaN values with 0 share_wz = share_wz.fillna(0.0) @@ -170,7 +173,7 @@ def calc_load_curves_cts(scenario): ) # Calculate shares of cts branches per nuts3-region - nuts3_share_wz = demands_nuts.groupby("nuts3").apply( + nuts3_share_wz = demands_nuts.groupby("nuts3", as_index=False).apply( lambda grp: grp / grp.sum() ) @@ -190,14 +193,16 @@ def calc_load_curves_cts(scenario): # Calculate shares of cts branches per hvmv substation share_subst = ( - demands_zensus.drop("demand", axis=1).groupby("bus_id").mean() + demands_zensus.drop(["nuts3", "demand"], axis=1) + .groupby("bus_id") + .mean() ) # Calculate cts annual demand per hvmv substation annual_demand_subst = demands_zensus.groupby("bus_id").demand.sum() # Return electrical load curves per hvmv substation - return calc_load_curve(share_subst, annual_demand_subst) + return calc_load_curve(share_subst, scenario, annual_demand_subst) def insert_cts_load(): @@ -215,8 +220,7 @@ def insert_cts_load(): create_table() - for scenario in ["eGon2035", "eGon100RE"]: - + for scenario in egon.data.config.settings()["egon-data"]["--scenarios"]: # Delete existing data from database db.execute_sql( f""" diff --git a/src/egon/data/datasets/electricity_demand_etrago.py b/src/egon/data/datasets/electricity_demand_etrago.py index 65b50a2a0..5c21efde5 100644 --- a/src/egon/data/datasets/electricity_demand_etrago.py +++ b/src/egon/data/datasets/electricity_demand_etrago.py @@ -31,16 +31,12 @@ def demands_per_bus(scenario): # Select data on CTS electricity demands per bus cts_curves = db.select_dataframe( - f"""SELECT bus_id, p_set FROM + f"""SELECT bus_id AS bus, p_set FROM {sources['cts_curves']['schema']}. {sources['cts_curves']['table']} WHERE scn_name = '{scenario}'""", - index_col="bus_id", ) - # Rename index - cts_curves.index.rename("bus", inplace=True) - # Select data on industrial demands assigned to osm landuse areas ind_curves_osm = db.select_dataframe( @@ -48,7 +44,6 @@ def demands_per_bus(scenario): {sources['osm_curves']['schema']}. {sources['osm_curves']['table']} WHERE scn_name = '{scenario}'""", - index_col="bus", ) # Select data on industrial demands assigned to industrial sites @@ -58,24 +53,23 @@ def demands_per_bus(scenario): {sources['sites_curves']['schema']}. {sources['sites_curves']['table']} WHERE scn_name = '{scenario}'""", - index_col="bus", ) # Select data on household electricity demands per bus hh_curves = db.select_dataframe( - f"""SELECT bus_id, p_set FROM + f"""SELECT bus_id AS bus, p_set FROM {sources['household_curves']['schema']}. {sources['household_curves']['table']} WHERE scn_name = '{scenario}'""", - index_col="bus_id", ) # Create one df by appending all imported dataframes - demand_curves = cts_curves.append( - [ind_curves_osm, ind_curves_sites, hh_curves] - ) + demand_curves = pd.concat( + [cts_curves, ind_curves_osm, ind_curves_sites, hh_curves], + ignore_index=True, + ).set_index("bus") # Split array to single columns in the dataframe demand_curves_split = demand_curves @@ -173,8 +167,7 @@ def export_to_db(): sources = egon.data.config.datasets()["etrago_electricity"]["sources"] targets = egon.data.config.datasets()["etrago_electricity"]["targets"] - for scenario in ["eGon2035", "eGon100RE"]: - + for scenario in egon.data.config.settings()["egon-data"]["--scenarios"]: # Delete existing data from database db.execute_sql( f""" @@ -271,7 +264,7 @@ class ElectricalLoadEtrago(Dataset): def __init__(self, dependencies): super().__init__( name="Electrical_load_etrago", - version="0.0.6", + version="0.0.8", dependencies=dependencies, tasks=(export_to_db,), ) diff --git a/src/egon/data/datasets/electricity_demand_timeseries/cts_buildings.py b/src/egon/data/datasets/electricity_demand_timeseries/cts_buildings.py index 748ddac94..b16f709fb 100644 --- a/src/egon/data/datasets/electricity_demand_timeseries/cts_buildings.py +++ b/src/egon/data/datasets/electricity_demand_timeseries/cts_buildings.py @@ -12,14 +12,14 @@ from geoalchemy2 import Geometry from geoalchemy2.shape import to_shape from psycopg2.extensions import AsIs, register_adapter -from sqlalchemy import REAL, Column, Integer, String, func +from sqlalchemy import REAL, Column, Integer, String, func, cast from sqlalchemy.ext.declarative import declarative_base import geopandas as gpd import numpy as np import pandas as pd import saio -from egon.data import db +from egon.data import config, db from egon.data import logger as log from egon.data.datasets import Dataset from egon.data.datasets.electricity_demand import ( @@ -279,7 +279,7 @@ def __init__(self, dependencies): version=self.version, dependencies=dependencies, tasks=( - cts_buildings, + cts_buildings, # TODO: status2023, currently fixed for only 2023 {cts_electricity, cts_heat}, get_cts_electricity_peak_load, map_all_used_buildings, @@ -319,7 +319,7 @@ def amenities_without_buildings(): ) .filter( EgonDemandRegioZensusElectricity.sector == "service", - EgonDemandRegioZensusElectricity.scenario == "eGon2035", + EgonDemandRegioZensusElectricity.scenario == "status2019", ) ) @@ -416,17 +416,17 @@ def create_synthetic_buildings(df, points=None, crs="EPSG:3035"): # TODO remove after #772 implementation of egon_building_id df.rename(columns={"id": "egon_building_id"}, inplace=True) - # get max number of building ids from synthetic residential table + # get max number of building ids from synthetic table with db.session_scope() as session: - max_synth_residential_id = session.execute( - func.max(OsmBuildingsSynthetic.id) + max_synth_building_id = session.execute( + func.max(cast(OsmBuildingsSynthetic.id, Integer)) ).scalar() - max_synth_residential_id = int(max_synth_residential_id) + max_synth_building_id = int(max_synth_building_id) # create sequential ids df["egon_building_id"] = range( - max_synth_residential_id + 1, - max_synth_residential_id + df.shape[0] + 1, + max_synth_building_id + 1, + max_synth_building_id + df.shape[0] + 1, ) df["area"] = df["geom_building"].area @@ -480,7 +480,7 @@ def buildings_with_amenities(): ) .filter( EgonDemandRegioZensusElectricity.sector == "service", - EgonDemandRegioZensusElectricity.scenario == "eGon2035", + EgonDemandRegioZensusElectricity.scenario == "status2023", # TODO: status2023 ) ) df_amenities_in_buildings = pd.read_sql( @@ -496,7 +496,6 @@ def buildings_with_amenities(): # retrieve building centroid bus_id with db.session_scope() as session: - cells_query = session.query( egon_map_zensus_buildings_filtered_all.id, MapZensusGridDistricts.bus_id.label("building_bus_id"), @@ -621,7 +620,6 @@ def buildings_without_amenities(): # buildings_filtered in cts-demand-cells without amenities with db.session_scope() as session: - # Synthetic Buildings q_synth_buildings = session.query( osm_buildings_synthetic.cell_id.cast(Integer).label( @@ -666,7 +664,7 @@ def buildings_without_amenities(): ) .filter( EgonDemandRegioZensusElectricity.sector == "service", - EgonDemandRegioZensusElectricity.scenario == "eGon2035", + EgonDemandRegioZensusElectricity.scenario == "status2019", ) .filter( EgonDemandRegioZensusElectricity.zensus_population_id.notin_( @@ -766,7 +764,7 @@ def cells_with_cts_demand_only(df_buildings_without_amenities): ) .filter( EgonDemandRegioZensusElectricity.sector == "service", - EgonDemandRegioZensusElectricity.scenario == "eGon2035", + EgonDemandRegioZensusElectricity.scenario == "status2019", ) .filter( EgonDemandRegioZensusElectricity.zensus_population_id.notin_( @@ -1213,10 +1211,11 @@ def cts_buildings(): Note: ----- Cells with CTS demand, amenities and buildings do not change within - the scenarios, only the demand itself. Therefore scenario eGon2035 + the scenarios, only the demand itself. Therefore scenario status2019 can be used universally to determine the cts buildings but not for the demand share. """ + # ========== Register np datatypes with SQLA ========== def adapt_numpy_float64(numpy_float64): return AsIs(numpy_float64) @@ -1230,7 +1229,7 @@ def adapt_numpy_int64(numpy_int64): log.info("Start logging!") # Buildings with amenities - df_buildings_with_amenities, df_lost_cells = buildings_with_amenities() + df_buildings_with_amenities, df_lost_cells = buildings_with_amenities() # TODO: status2023 this is fixed to 2023 log.info("Buildings with amenities selected!") # Median number of amenities per cell @@ -1249,11 +1248,10 @@ def adapt_numpy_int64(numpy_int64): # Amenities not assigned to buildings df_amenities_without_buildings = amenities_without_buildings() - log.info("Amenities without buildlings selected!") + log.info("Amenities without buildings selected!") # Append lost cells due to duplicated ids, to cover all demand cells if not df_lost_cells.empty: - # Number of synth amenities per cell df_lost_cells["amenities"] = median_n_amenities # create row for every amenity @@ -1262,8 +1260,8 @@ def adapt_numpy_int64(numpy_int64): ) df_lost_cells = df_lost_cells.explode("amenities") df_lost_cells.drop(columns="amenities", inplace=True) - df_amenities_without_buildings = df_amenities_without_buildings.append( - df_lost_cells, ignore_index=True + df_amenities_without_buildings = pd.concat( + [df_amenities_without_buildings, df_lost_cells], ignore_index=True ) log.info( f"{df_lost_cells.shape[0]} lost cells due to substation " @@ -1277,7 +1275,7 @@ def adapt_numpy_int64(numpy_int64): df_amenities_without_buildings, points="geom_amenity" ) log.info("Synthetic buildings created!") - + # df_synthetic_buildings_with_amenities["scn_name"] = scn_name # TODO: status 2023, add eventually # TODO remove renaming after #722 write_table_to_postgis( df_synthetic_buildings_with_amenities.rename( @@ -1417,21 +1415,21 @@ def cts_electricity(): cells_query.statement, cells_query.session.bind, index_col=None ) log.info("CTS buildings from DB imported!") - df_demand_share_2035 = calc_building_demand_profile_share( - df_cts_buildings, scenario="eGon2035", sector="electricity" - ) - log.info("Profile share for egon2035 calculated!") - df_demand_share_100RE = calc_building_demand_profile_share( - df_cts_buildings, scenario="eGon100RE", sector="electricity" - ) - log.info("Profile share for egon100RE calculated!") + df_demand_share = pd.DataFrame() + + for scenario in config.settings()["egon-data"]["--scenarios"]: + df_demand_share_per_scenario = calc_building_demand_profile_share( + df_cts_buildings, scenario=scenario, sector="electricity" + ) + log.info(f"Profile share for {scenario} calculated!") + + df_demand_share = pd.concat( + [df_demand_share, df_demand_share_per_scenario], + axis=0, + ignore_index=True, + ) - df_demand_share = pd.concat( - [df_demand_share_2035, df_demand_share_100RE], - axis=0, - ignore_index=True, - ) df_demand_share.rename(columns={"id": "building_id"}, inplace=True) write_table_to_postgres( @@ -1456,19 +1454,19 @@ def cts_heat(): ) log.info("CTS buildings from DB imported!") - df_demand_share_2035 = calc_building_demand_profile_share( - df_cts_buildings, scenario="eGon2035", sector="heat" - ) - log.info("Profile share for egon2035 calculated!") - df_demand_share_100RE = calc_building_demand_profile_share( - df_cts_buildings, scenario="eGon100RE", sector="heat" - ) - log.info("Profile share for egon100RE calculated!") - df_demand_share = pd.concat( - [df_demand_share_2035, df_demand_share_100RE], - axis=0, - ignore_index=True, - ) + df_demand_share = pd.DataFrame() + + for scenario in config.settings()["egon-data"]["--scenarios"]: + df_demand_share_per_scenario = calc_building_demand_profile_share( + df_cts_buildings, scenario=scenario, sector="heat" + ) + log.info(f"Profile share for {scenario} calculated!") + + df_demand_share = pd.concat( + [df_demand_share, df_demand_share_per_scenario], + axis=0, + ignore_index=True, + ) df_demand_share.rename(columns={"id": "building_id"}, inplace=True) @@ -1496,8 +1494,7 @@ def get_cts_electricity_peak_load(): ).delete() log.info("Cts electricity peak load removed from DB!") - for scenario in ["eGon2035", "eGon100RE"]: - + for scenario in config.settings()["egon-data"]["--scenarios"]: with db.session_scope() as session: cells_query = session.query( EgonCtsElectricityDemandBuildingShare @@ -1568,8 +1565,7 @@ def get_cts_heat_peak_load(): ).delete() log.info("Cts heat peak load removed from DB!") - for scenario in ["eGon2035", "eGon100RE"]: - + for scenario in config.settings()["egon-data"]["--scenarios"]: with db.session_scope() as session: cells_query = session.query( EgonCtsElectricityDemandBuildingShare diff --git a/src/egon/data/datasets/electricity_demand_timeseries/hh_buildings.py b/src/egon/data/datasets/electricity_demand_timeseries/hh_buildings.py index 89ac83c50..10d9a58b7 100755 --- a/src/egon/data/datasets/electricity_demand_timeseries/hh_buildings.py +++ b/src/egon/data/datasets/electricity_demand_timeseries/hh_buildings.py @@ -64,6 +64,7 @@ class OsmBuildingsSynthetic(Base): id = Column(String, primary_key=True) cell_id = Column(String, index=True) + # scn_name = Column(String, index=True) # TODO: status2023 currently fixed to 2023 geom_building = Column(Geometry("Polygon", 3035), index=True) geom_point = Column(Geometry("POINT", 3035)) n_amenities_inside = Column(Integer) @@ -259,7 +260,7 @@ def generate_synthetic_buildings(missing_buildings, edge_length): destatis_zensus_population_per_ha_inside_germany ).filter( destatis_zensus_population_per_ha_inside_germany.c.id.in_( - missing_buildings.index + missing_buildings.index.unique() ) ) @@ -281,7 +282,7 @@ def generate_synthetic_buildings(missing_buildings, edge_length): columns={ "building_count": "building_id", "cell_profile_ids": "profiles", - "index": "cell_id", + "id": "cell_id", } ) @@ -560,6 +561,8 @@ def get_building_peak_loads(): session.query( HouseholdElectricityProfilesOfBuildings, HouseholdElectricityProfilesInCensusCells.nuts3, + HouseholdElectricityProfilesInCensusCells.factor_2019, + HouseholdElectricityProfilesInCensusCells.factor_2023, HouseholdElectricityProfilesInCensusCells.factor_2035, HouseholdElectricityProfilesInCensusCells.factor_2050, ) @@ -574,6 +577,9 @@ def get_building_peak_loads(): cells_query.statement, cells_query.session.bind, index_col="id" ) + # fill columns with None with np.nan to allow multiplication with emtpy columns + df_buildings_and_profiles = df_buildings_and_profiles.fillna(np.nan) + # Read demand profiles from egon-data-bundle df_profiles = get_iee_hh_demand_profiles_raw() @@ -601,16 +607,22 @@ def ve(s): names=("profile_id", "building_id"), ) df_building_peak_load_nuts3.columns = m_index - df_building_peak_load_nuts3 = df_building_peak_load_nuts3.sum( - level="building_id", axis=1 - ).max() + df_building_peak_load_nuts3 = ( + df_building_peak_load_nuts3.groupby("building_id", axis=1) + .sum() + .max() + ) df_building_peak_load_nuts3 = pd.DataFrame( [ + df_building_peak_load_nuts3 * df["factor_2019"].unique(), + df_building_peak_load_nuts3 * df["factor_2023"].unique(), df_building_peak_load_nuts3 * df["factor_2035"].unique(), df_building_peak_load_nuts3 * df["factor_2050"].unique(), ], index=[ + "status2019", + "status2023", "eGon2035", "eGon100RE", ], diff --git a/src/egon/data/datasets/electricity_demand_timeseries/hh_profiles.py b/src/egon/data/datasets/electricity_demand_timeseries/hh_profiles.py index 6598e7695..90a828127 100644 --- a/src/egon/data/datasets/electricity_demand_timeseries/hh_profiles.py +++ b/src/egon/data/datasets/electricity_demand_timeseries/hh_profiles.py @@ -13,7 +13,7 @@ import os import random -from airflow.operators.python_operator import PythonOperator +from airflow.operators.python import PythonOperator from sqlalchemy import ARRAY, Column, Float, Integer, String from sqlalchemy.dialects.postgresql import CHAR, INTEGER, REAL from sqlalchemy.ext.declarative import declarative_base @@ -22,6 +22,7 @@ from egon.data import db from egon.data.datasets import Dataset +from egon.data.datasets.scenario_parameters import get_scenario_year from egon.data.datasets.zensus_mv_grid_districts import MapZensusGridDistricts import egon.data.config @@ -63,6 +64,8 @@ class HouseholdElectricityProfilesInCensusCells(Base): cell_profile_ids = Column(ARRAY(String, dimensions=1)) nuts3 = Column(String) nuts1 = Column(String) + factor_2019 = Column(Float) + factor_2023 = Column(Float) factor_2035 = Column(Float) factor_2050 = Column(Float) @@ -231,28 +234,63 @@ class HouseholdDemands(Dataset): version: str = "0.0.10" def __init__(self, dependencies): - mv_hh_electricity_load_2035 = PythonOperator( - task_id="MV-hh-electricity-load-2035", - python_callable=mv_grid_district_HH_electricity_load, - op_args=["eGon2035", 2035], - op_kwargs={"drop_table": True}, - ) + tasks = (houseprofiles_in_census_cells,) + + if ( + "status2019" + in egon.data.config.settings()["egon-data"]["--scenarios"] + ): + mv_hh_electricity_load_2035 = PythonOperator( + task_id="MV-hh-electricity-load-2019", + python_callable=mv_grid_district_HH_electricity_load, + op_args=["status2019", 2019], + op_kwargs={"drop_table": True}, + ) - mv_hh_electricity_load_2050 = PythonOperator( - task_id="MV-hh-electricity-load-2050", - python_callable=mv_grid_district_HH_electricity_load, - op_args=["eGon100RE", 2050], - ) + tasks = tasks + (mv_hh_electricity_load_2035,) + + if ( + "status2023" + in egon.data.config.settings()["egon-data"]["--scenarios"] + ): + mv_hh_electricity_load_2035 = PythonOperator( + task_id="MV-hh-electricity-load-2023", + python_callable=mv_grid_district_HH_electricity_load, + op_args=["status2023", 2023], + op_kwargs={"drop_table": True}, + ) + + tasks = tasks + (mv_hh_electricity_load_2035,) + + if ( + "eGon2035" + in egon.data.config.settings()["egon-data"]["--scenarios"] + ): + mv_hh_electricity_load_2035 = PythonOperator( + task_id="MV-hh-electricity-load-2035", + python_callable=mv_grid_district_HH_electricity_load, + op_args=["eGon2035", 2035], + ) + + tasks = tasks + (mv_hh_electricity_load_2035,) + + if ( + "eGon100RE" + in egon.data.config.settings()["egon-data"]["--scenarios"] + ): + mv_hh_electricity_load_2050 = PythonOperator( + task_id="MV-hh-electricity-load-2050", + python_callable=mv_grid_district_HH_electricity_load, + op_args=["eGon100RE", 2050], + ) + + tasks = tasks + (mv_hh_electricity_load_2050,) super().__init__( name=self.name, version=self.version, dependencies=dependencies, - tasks=( - houseprofiles_in_census_cells, - mv_hh_electricity_load_2035, - mv_hh_electricity_load_2050, - ), + tasks=tasks, ) @@ -930,7 +968,7 @@ def impute_missing_hh_in_populated_cells(df_census_households_grid): * df_wo_hh_population_i.shape[0] ).values # append new cells - df_w_hh = df_w_hh.append(df_repeated, ignore_index=True) + df_w_hh = pd.concat([df_w_hh, df_repeated], ignore_index=True) return df_w_hh @@ -1196,12 +1234,13 @@ def refine_census_data_at_cell_level( df_group, dist_households_nuts1, hh_10types_cluster ) df_distribution_group["characteristics_code"] = hh_5type_cluster - df_distribution_nuts1 = df_distribution_nuts1.append( - df_distribution_group + df_distribution_nuts1 = pd.concat( + [df_distribution_nuts1, df_distribution_group], + ignore_index=True, ) - df_distribution_nuts0 = df_distribution_nuts0.append( - df_distribution_nuts1 + df_distribution_nuts0 = pd.concat( + [df_distribution_nuts0, df_distribution_nuts1], ignore_index=True ) df_census_households_grid_refined = df_census_households_grid.merge( @@ -1322,7 +1361,6 @@ def assign_hh_demand_profiles_to_cells(df_zensus_cells, df_iee_profiles): # only use non zero entries df_zensus_cells = df_zensus_cells.loc[df_zensus_cells["hh_10types"] != 0] for grid_id, df_cell in df_zensus_cells.groupby(by="grid_id"): - # random sampling of household profiles for each cell # with or without replacement (see :func:`get_cell_demand_profile_ids`) # within cell but after number of households are rounded to the nearest @@ -1393,16 +1431,16 @@ def adjust_to_demand_regio_nuts3_annual( # ############## # demand regio in MWh # profiles in Wh - df_hh_profiles_in_census_cells.loc[nuts3_cell_ids, "factor_2035"] = ( - df_demand_regio.loc[(2035, nuts3_id), "demand_mwha"] - * 1e3 - / (nuts3_profiles_sum_annual / 1e3) - ) - df_hh_profiles_in_census_cells.loc[nuts3_cell_ids, "factor_2050"] = ( - df_demand_regio.loc[(2050, nuts3_id), "demand_mwha"] - * 1e3 - / (nuts3_profiles_sum_annual / 1e3) - ) + + for scn in egon.data.config.settings()["egon-data"]["--scenarios"]: + year = get_scenario_year(scn) + df_hh_profiles_in_census_cells.loc[ + nuts3_cell_ids, f"factor_{year}" + ] = ( + df_demand_regio.loc[(year, nuts3_id), "demand_mwha"] + * 1e3 + / (nuts3_profiles_sum_annual / 1e3) + ) return df_hh_profiles_in_census_cells @@ -1765,16 +1803,39 @@ def get_hh_profiles_from_db(profile_ids): return df_profile_loads +def get_demand_regio_hh_profiles_from_db(year): + """ + Retrieve demand regio household electricity demand profiles in nuts3 level + + Parameters + ---------- + year: int + To which year belong the required demand profile + + Returns + ------- + pd.DataFrame + Selection of household demand profiles + """ + + query = """Select * from demand.demandregio_household_load_profiles + Where year = year""" + + df_profile_loads = pd.read_sql( + query, db.engine(), index_col="id" + ) + + return df_profile_loads def mv_grid_district_HH_electricity_load( - scenario_name, scenario_year, drop_table=False + scenario_name, scenario_year, drop_table ): """ Aggregated household demand time series at HV/MV substation level Calculate the aggregated demand time series based on the demand profiles of each zensus cell inside each MV grid district. Profiles are read from - local hdf5-file. + local hdf5-file or demand timeseries per nuts3 in db. Creates table `demand.egon_etrago_electricity_households` with Household electricity demand profiles aggregated at MV grid district level in MWh. Primarily used to create the eTraGo data model. @@ -1816,33 +1877,82 @@ def tuple_format(x): cells_query.statement, cells_query.session.bind, index_col="cell_id" ) - # convert profile ids to tuple (type, id) format - cells["cell_profile_ids"] = cells["cell_profile_ids"].apply( - lambda x: list(map(tuple_format, x)) - ) + method = egon.data.config.settings()["egon-data"][ + "--household-demand-source" + ] - # Read demand profiles from egon-data-bundle - df_iee_profiles = get_iee_hh_demand_profiles_raw() + if method == "demand-regio": + #Import demand regio timeseries demand per nuts3 area + dr_series = pd.read_sql_query(""" + SELECT year, nuts3, load_in_mwh FROM demand.demandregio_household_load_profiles + """, + con = engine + ) + dr_series = dr_series[dr_series["year"] == scenario_year] + dr_series.drop(columns=["year"], inplace=True) + dr_series.set_index("nuts3", inplace=True) + dr_series = dr_series.squeeze() + + #Population data per cell_id is used to scale the demand per nuts3 + population = pd.read_sql_query(""" + SELECT grid_id, population FROM society.destatis_zensus_population_per_ha + """, + con = engine + ) + population.set_index("grid_id", inplace=True) + population = population.squeeze() + population.loc[population==-1] = 0 - # Process profiles for further use - df_iee_profiles = set_multiindex_to_profiles(df_iee_profiles) + cells["population"] = cells["grid_id"].map(population) + + factor_column = f"""factor_{scenario_year}""" + + mvgd_profiles = pd.DataFrame( + columns=["p_set", "q_set"], index=cells.bus_id.unique() + ) + mvgd_profiles.index.name = "bus_id" - # Create aggregated load profile for each MV grid district - mvgd_profiles_dict = {} - for grid_district, data in cells.groupby("bus_id"): - mvgd_profile = get_load_timeseries( - df_iee_profiles=df_iee_profiles, - df_hh_profiles_in_census_cells=data, - cell_ids=data.index, - year=scenario_year, - peak_load_only=False, + for nuts3, df in cells.groupby("nuts3"): + cells.loc[df.index, factor_column] = df["population"] / df["population"].sum() + + for bus, df_bus in cells.groupby("bus_id"): + load_nuts = [0] * 8760 + for nuts3, df_nuts in df_bus.groupby("nuts3"): + factor_nuts = df_nuts[factor_column].sum() + total_load = [x * factor_nuts for x in dr_series[nuts3]] + load_nuts = [sum(x) for x in zip(load_nuts, total_load)] + mvgd_profiles.at[bus, "p_set"] = load_nuts + + mvgd_profiles.reset_index(inplace=True) + + elif method == "IEE": + # convert profile ids to tuple (type, id) format + cells["cell_profile_ids"] = cells["cell_profile_ids"].apply( + lambda x: list(map(tuple_format, x)) ) - mvgd_profiles_dict[grid_district] = [mvgd_profile.round(3).to_list()] - mvgd_profiles = pd.DataFrame.from_dict(mvgd_profiles_dict, orient="index") - # Reshape data: put MV grid ids in columns to a single index column - mvgd_profiles = mvgd_profiles.reset_index() - mvgd_profiles.columns = ["bus_id", "p_set"] + # Read demand profiles from egon-data-bundle + df_iee_profiles = get_iee_hh_demand_profiles_raw() + + # Process profiles for further use + df_iee_profiles = set_multiindex_to_profiles(df_iee_profiles) + + # Create aggregated load profile for each MV grid district + mvgd_profiles_dict = {} + for grid_district, data in cells.groupby("bus_id"): + mvgd_profile = get_load_timeseries( + df_iee_profiles=df_iee_profiles, + df_hh_profiles_in_census_cells=data, + cell_ids=data.index, + year=scenario_year, + peak_load_only=False, + ) + mvgd_profiles_dict[grid_district] = [mvgd_profile.round(3).to_list()] + mvgd_profiles = pd.DataFrame.from_dict(mvgd_profiles_dict, orient="index") + + # Reshape data: put MV grid ids in columns to a single index column + mvgd_profiles = mvgd_profiles.reset_index() + mvgd_profiles.columns = ["bus_id", "p_set"] # Add remaining columns mvgd_profiles["scn_name"] = scenario_name @@ -1865,7 +1975,6 @@ def tuple_format(x): index=False, ) - def get_scaled_profiles_from_db( attribute, list_of_identifiers, year, aggregate=True, peak_load_only=False ): diff --git a/src/egon/data/datasets/emobility/motorized_individual_travel/__init__.py b/src/egon/data/datasets/emobility/motorized_individual_travel/__init__.py index b73454fe4..dba8a14c4 100644 --- a/src/egon/data/datasets/emobility/motorized_individual_travel/__init__.py +++ b/src/egon/data/datasets/emobility/motorized_individual_travel/__init__.py @@ -16,12 +16,12 @@ import os import tarfile -from airflow.operators.python_operator import PythonOperator +from airflow.operators.python import PythonOperator from psycopg2.extensions import AsIs, register_adapter import numpy as np import pandas as pd -from egon.data import db, subprocess +from egon.data import config, db, subprocess from egon.data.datasets import Dataset from egon.data.datasets.emobility.motorized_individual_travel.db_classes import ( # noqa: E501 EgonEvCountMunicipality, @@ -31,6 +31,7 @@ EgonEvMvGridDistrict, EgonEvPool, EgonEvTrip, + add_metadata, ) from egon.data.datasets.emobility.motorized_individual_travel.ev_allocation import ( # noqa: E501 allocate_evs_numbers, @@ -48,6 +49,8 @@ from egon.data.datasets.emobility.motorized_individual_travel.model_timeseries import ( # noqa: E501 delete_model_data_from_db, generate_model_data_bunch, + generate_model_data_status2019_remaining, + generate_model_data_status2023_remaining, generate_model_data_eGon100RE_remaining, generate_model_data_eGon2035_remaining, read_simbev_metadata_file, @@ -142,8 +145,8 @@ def download_and_preprocess(): kba_data[ ["ags_reg_district", "reg_district"] ] = kba_data.reg_district.str.split( - " ", - 1, + pat=" ", + n=1, expand=True, ) kba_data.ags_reg_district = kba_data.ags_reg_district.astype("int") @@ -180,7 +183,7 @@ def extract_trip_file(): """Extract trip file from data bundle""" trip_dir = DATA_BUNDLE_DIR / Path("mit_trip_data") - for scenario_name in ["eGon2035", "eGon100RE"]: + for scenario_name in config.settings()["egon-data"]["--scenarios"]: print(f"SCENARIO: {scenario_name}") trip_file = trip_dir / Path( DATASET_CFG["original_data"]["sources"]["trips"][scenario_name][ @@ -208,7 +211,7 @@ def import_csv(f): df["simbev_ev_id"] = "_".join(f.name.split("_")[0:3]) return df - for scenario_name in ["eGon2035", "eGon100RE"]: + for scenario_name in config.settings()["egon-data"]["--scenarios"]: print(f"SCENARIO: {scenario_name}") trip_dir_name = Path( DATASET_CFG["original_data"]["sources"]["trips"][scenario_name][ @@ -255,7 +258,7 @@ def import_csv(f): # Split simBEV id into type and id evs_unique[["type", "simbev_ev_id"]] = evs_unique[ "simbev_ev_id" - ].str.rsplit("_", 1, expand=True) + ].str.rsplit(pat="_", n=1, expand=True) evs_unique.simbev_ev_id = evs_unique.simbev_ev_id.astype(int) evs_unique["scenario"] = scenario_name @@ -308,14 +311,14 @@ def write_metadata_to_db(): "scenario": str, "eta_cp": float, "stepsize": int, - "start_date": np.datetime64, - "end_date": np.datetime64, + "start_date": "datetime64[ns]", + "end_date": "datetime64[ns]", "soc_min": float, "grid_timeseries": bool, "grid_timeseries_by_usecase": bool, } - for scenario_name in ["eGon2035", "eGon100RE"]: + for scenario_name in config.settings()["egon-data"]["--scenarios"]: meta_run_config = read_simbev_metadata_file( scenario_name, "config" ).loc["basic"] @@ -446,34 +449,45 @@ def generate_model_data_tasks(scenario_name): ) ) - if scenario_name == "eGon2035": + if scenario_name == "status2019": + tasks.add(generate_model_data_status2019_remaining) + if scenario_name == "status2023": + tasks.add(generate_model_data_status2023_remaining) + elif scenario_name == "eGon2035": tasks.add(generate_model_data_eGon2035_remaining) elif scenario_name == "eGon100RE": tasks.add(generate_model_data_eGon100RE_remaining) return tasks + tasks = ( + create_tables, + { + ( + download_and_preprocess, + allocate_evs_numbers, + ), + ( + extract_trip_file, + write_metadata_to_db, + write_evs_trips_to_db, + ), + }, + allocate_evs_to_grid_districts, + delete_model_data_from_db, + ) + + tasks_per_scenario = set() + + for scenario_name in config.settings()["egon-data"]["--scenarios"]: + tasks_per_scenario.update( + generate_model_data_tasks(scenario_name=scenario_name) + ) + + tasks = tasks + (tasks_per_scenario,) + super().__init__( name=self.name, version=self.version, dependencies=dependencies, - tasks=( - create_tables, - { - ( - download_and_preprocess, - allocate_evs_numbers, - ), - ( - extract_trip_file, - write_metadata_to_db, - write_evs_trips_to_db, - ), - }, - allocate_evs_to_grid_districts, - delete_model_data_from_db, - { - *generate_model_data_tasks(scenario_name="eGon2035"), - *generate_model_data_tasks(scenario_name="eGon100RE"), - }, - ), + tasks=tasks, ) diff --git a/src/egon/data/datasets/emobility/motorized_individual_travel/db_classes.py b/src/egon/data/datasets/emobility/motorized_individual_travel/db_classes.py index 95744835b..4fe967239 100644 --- a/src/egon/data/datasets/emobility/motorized_individual_travel/db_classes.py +++ b/src/egon/data/datasets/emobility/motorized_individual_travel/db_classes.py @@ -1,7 +1,10 @@ """ DB tables / SQLAlchemy ORM classes for motorized individual travel """ +import datetime +import json +from omi.dialects import get_dialect from sqlalchemy import ( Boolean, Column, @@ -15,11 +18,23 @@ from sqlalchemy.dialects.postgresql import REAL from sqlalchemy.ext.declarative import declarative_base +from egon.data import db +from egon.data.datasets.emobility.motorized_individual_travel.helpers import ( + read_simbev_metadata_file, +) from egon.data.datasets.mv_grid_districts import MvGridDistricts from egon.data.datasets.scenario_parameters import EgonScenario - -# from sqlalchemy.orm import relationship - +from egon.data.metadata import ( + context, + contributors, + generate_resource_fields_from_db_table, + license_agpl, + license_ccby, + license_odbl, + meta_metadata, + meta_metadata, + sources, +) Base = declarative_base() @@ -270,7 +285,7 @@ class EgonEvMetadata(Base): __tablename__ = "egon_ev_metadata" __table_args__ = {"schema": "demand"} - scenario = Column(String, primary_key=True) + scenario = Column(String, primary_key=True, index=True) eta_cp = Column(Float) stepsize = Column(Integer) start_date = Column(DateTime) @@ -278,3 +293,373 @@ class EgonEvMetadata(Base): soc_min = Column(Float) grid_timeseries = Column(Boolean) grid_timeseries_by_usecase = Column(Boolean) + + +def add_metadata(): + """ + Add metadata to tables egon_ev_metadata, egon_ev_mv_grid_district, + egon_ev_trip in schema demand + """ + # egon_ev_metadata + schema = "demand" + meta_run_config = read_simbev_metadata_file("eGon100RE", "config").loc[ + "basic" + ] + + contris = contributors(["kh", "kh"]) + + contris[0]["date"] = "2023-03-17" + + contris[0]["object"] = "metadata" + contris[1]["object"] = "dataset" + + contris[0]["comment"] = "Add metadata to dataset." + contris[1]["comment"] = "Add workflow to generate dataset." + + table = "egon_ev_metadata" + name = f"{schema}.{table}" + + meta = { + "name": name, + "title": "eGon EV metadata", + "id": "WILL_BE_SET_AT_PUBLICATION", + "description": ( + "Metadata regarding the generation of EV trip profiles with SimBEV" + ), + "language": "en-US", + "keywords": ["ev", "mit", "simbev", "metadata", "parameters"], + "publicationDate": datetime.date.today().isoformat(), + "context": context(), + "spatial": { + "location": "none", + "extent": "none", + "resolution": "none", + }, + "temporal": { + "referenceDate": f"{meta_run_config.start_date}", + "timeseries": {}, + }, + "sources": [ + sources()["egon-data"], + { + "title": "SimBEV", + "description": ( + "Simulation of electric vehicle charging demand" + ), + "path": "https://github.com/rl-institut/simbev", + "licenses": [ + license_ccby(attribution="© Reiner Lemoine Institut") + ], + }, + { + "title": "SimBEV", + "description": ( + "Simulation of electric vehicle charging demand" + ), + "path": "https://github.com/rl-institut/simbev", + "licenses": [ + license_agpl(attribution="© Reiner Lemoine Institut") + ], + }, + ], + "licenses": [license_ccby()], + "contributors": contris, + "resources": [ + { + "profile": "tabular-data-resource", + "name": name, + "path": "None", + "format": "PostgreSQL", + "encoding": "UTF-8", + "schema": { + "fields": generate_resource_fields_from_db_table( + schema, + table, + ), + "primaryKey": "scenario", + }, + "dialect": {"delimiter": "", "decimalSeparator": ""}, + } + ], + "review": {"path": "", "badge": ""}, + "metaMetadata": meta_metadata(), + "_comment": { + "metadata": ( + "Metadata documentation and explanation (https://github." + "com/OpenEnergyPlatform/oemetadata/blob/master/metadata/" + "v141/metadata_key_description.md)" + ), + "dates": ( + "Dates and time must follow the ISO8601 including time " + "zone (YYYY-MM-DD or YYYY-MM-DDThh:mm:ss±hh)" + ), + "units": "Use a space between numbers and units (100 m)", + "languages": ( + "Languages must follow the IETF (BCP47) format (en-GB, " + "en-US, de-DE)" + ), + "licenses": ( + "License name must follow the SPDX License List " + "(https://spdx.org/licenses/)" + ), + "review": ( + "Following the OEP Data Review (https://github.com/" + "OpenEnergyPlatform/data-preprocessing/wiki)" + ), + "none": "If not applicable use (none)", + }, + } + + dialect = get_dialect(f"oep-v{meta_metadata()['metadataVersion'][4:7]}")() + + meta = dialect.compile_and_render(dialect.parse(json.dumps(meta))) + + db.submit_comment( + f"'{json.dumps(meta)}'", + schema, + table, + ) + + table = "egon_ev_mv_grid_district" + name = f"{schema}.{table}" + + meta = { + "name": name, + "title": "eGon EV MV grid district", + "id": "WILL_BE_SET_AT_PUBLICATION", + "description": ("EV mapping to MV grids"), + "language": "en-US", + "keywords": ["ev", "mit", "simbev", "mv", "grid"], + "publicationDate": datetime.date.today().isoformat(), + "context": context(), + "spatial": { + "location": "none", + "extent": "Germany", + "resolution": "Grid district", + }, + "temporal": { + "referenceDate": f"{meta_run_config.start_date}", + "timeseries": {}, + }, + "sources": [ + sources()["bgr_inspee"], + sources()["bgr_inspeeds"], + sources()["bgr_inspeeds_data_bundle"], + sources()["bgr_inspeeds_report"], + sources()["demandregio"], + sources()["dsm-heitkoetter"], + sources()["egon-data"], + sources()["era5"], + sources()["hotmaps_industrial_sites"], + sources()["mastr"], + sources()["nep2021"], + sources()["openffe_gas"], + sources()["openstreetmap"], + sources()["peta"], + sources()["pipeline_classification"], + sources()["SciGRID_gas"], + sources()["schmidt"], + sources()["technology-data"], + sources()["tyndp"], + sources()["vg250"], + sources()["zensus"], + { + "title": "SimBEV", + "description": ( + "Simulation of electric vehicle charging demand" + ), + "path": "https://github.com/rl-institut/simbev", + "licenses": [ + license_ccby(attribution="© Reiner Lemoine Institut") + ], + }, + { + "title": "SimBEV", + "description": ( + "Simulation of electric vehicle charging demand" + ), + "path": "https://github.com/rl-institut/simbev", + "licenses": [ + license_agpl(attribution="© Reiner Lemoine Institut") + ], + }, + ], + "licenses": [license_odbl()], + "contributors": contris, + "resources": [ + { + "profile": "tabular-data-resource", + "name": name, + "path": "None", + "format": "PostgreSQL", + "encoding": "UTF-8", + "schema": { + "fields": generate_resource_fields_from_db_table( + schema, + table, + ), + "primaryKey": "id", + }, + "dialect": {"delimiter": "", "decimalSeparator": ""}, + } + ], + "review": {"path": "", "badge": ""}, + "metaMetadata": meta_metadata(), + "_comment": { + "metadata": ( + "Metadata documentation and explanation (https://github." + "com/OpenEnergyPlatform/oemetadata/blob/master/metadata/" + "v141/metadata_key_description.md)" + ), + "dates": ( + "Dates and time must follow the ISO8601 including time " + "zone (YYYY-MM-DD or YYYY-MM-DDThh:mm:ss±hh)" + ), + "units": "Use a space between numbers and units (100 m)", + "languages": ( + "Languages must follow the IETF (BCP47) format (en-GB, " + "en-US, de-DE)" + ), + "licenses": ( + "License name must follow the SPDX License List " + "(https://spdx.org/licenses/)" + ), + "review": ( + "Following the OEP Data Review (https://github.com/" + "OpenEnergyPlatform/data-preprocessing/wiki)" + ), + "none": "If not applicable use (none)", + }, + } + + dialect = get_dialect(f"oep-v{meta_metadata()['metadataVersion'][4:7]}")() + + meta = dialect.compile_and_render(dialect.parse(json.dumps(meta))) + + db.submit_comment( + f"'{json.dumps(meta)}'", + schema, + table, + ) + + table = "egon_ev_trip" + name = f"{schema}.{table}" + + meta = { + "name": name, + "title": "eGon EV trip profiles", + "id": "WILL_BE_SET_AT_PUBLICATION", + "description": ("EV trip profiles generated with SimBEV"), + "language": "en-US", + "keywords": ["ev", "mit", "simbev", "trip", "profiles"], + "publicationDate": datetime.date.today().isoformat(), + "context": context(), + "spatial": { + "location": "none", + "extent": "Germany", + "resolution": "none", + }, + "temporal": { + "referenceDate": f"{meta_run_config.start_date}", + "timeseries": {}, + }, + "sources": [ + sources()["bgr_inspee"], + sources()["bgr_inspeeds"], + sources()["bgr_inspeeds_data_bundle"], + sources()["bgr_inspeeds_report"], + sources()["demandregio"], + sources()["dsm-heitkoetter"], + sources()["egon-data"], + sources()["era5"], + sources()["hotmaps_industrial_sites"], + sources()["mastr"], + sources()["nep2021"], + sources()["openffe_gas"], + sources()["openstreetmap"], + sources()["peta"], + sources()["pipeline_classification"], + sources()["SciGRID_gas"], + sources()["schmidt"], + sources()["technology-data"], + sources()["tyndp"], + sources()["vg250"], + sources()["zensus"], + { + "title": "SimBEV", + "description": ( + "Simulation of electric vehicle charging demand" + ), + "path": "https://github.com/rl-institut/simbev", + "licenses": [ + license_ccby(attribution="© Reiner Lemoine Institut") + ], + }, + { + "title": "SimBEV", + "description": ( + "Simulation of electric vehicle charging demand" + ), + "path": "https://github.com/rl-institut/simbev", + "licenses": [ + license_agpl(attribution="© Reiner Lemoine Institut") + ], + }, + ], + "licenses": [license_odbl()], + "contributors": contris, + "resources": [ + { + "profile": "tabular-data-resource", + "name": name, + "path": "None", + "format": "PostgreSQL", + "encoding": "UTF-8", + "schema": { + "fields": generate_resource_fields_from_db_table( + schema, + table, + ), + "primaryKey": ["scenario", "event_id"], + }, + "dialect": {"delimiter": "", "decimalSeparator": ""}, + } + ], + "review": {"path": "", "badge": ""}, + "metaMetadata": meta_metadata(), + "_comment": { + "metadata": ( + "Metadata documentation and explanation (https://github." + "com/OpenEnergyPlatform/oemetadata/blob/master/metadata/" + "v141/metadata_key_description.md)" + ), + "dates": ( + "Dates and time must follow the ISO8601 including time " + "zone (YYYY-MM-DD or YYYY-MM-DDThh:mm:ss±hh)" + ), + "units": "Use a space between numbers and units (100 m)", + "languages": ( + "Languages must follow the IETF (BCP47) format (en-GB, " + "en-US, de-DE)" + ), + "licenses": ( + "License name must follow the SPDX License List " + "(https://spdx.org/licenses/)" + ), + "review": ( + "Following the OEP Data Review (https://github.com/" + "OpenEnergyPlatform/data-preprocessing/wiki)" + ), + "none": "If not applicable use (none)", + }, + } + + dialect = get_dialect(f"oep-v{meta_metadata()['metadataVersion'][4:7]}")() + + meta = dialect.compile_and_render(dialect.parse(json.dumps(meta))) + + db.submit_comment( + f"'{json.dumps(meta)}'", + schema, + table, + ) diff --git a/src/egon/data/datasets/emobility/motorized_individual_travel/ev_allocation.py b/src/egon/data/datasets/emobility/motorized_individual_travel/ev_allocation.py index 61c2ecfb4..6737ea756 100644 --- a/src/egon/data/datasets/emobility/motorized_individual_travel/ev_allocation.py +++ b/src/egon/data/datasets/emobility/motorized_individual_travel/ev_allocation.py @@ -12,7 +12,7 @@ import numpy as np import pandas as pd -from egon.data import db +from egon.data import db, config from egon.data.datasets.emobility.motorized_individual_travel.db_classes import ( EgonEvCountMunicipality, EgonEvCountMvGridDistrict, @@ -106,7 +106,7 @@ def fix_missing_ags_municipality_regiostar(muns, rs7_data): f"based upon AGS {ags}." ) similar_entry.ags = ags - rs7_data = rs7_data.append(similar_entry) + rs7_data = pd.concat([rs7_data, similar_entry]) print("Consider to update RS7.") # VG250 entries missing: elif name == "VG250": @@ -334,7 +334,7 @@ def calc_evs_per_grid_district(ev_data_muns): # * pop_mun_in_mvgd_of_mun_total: relative pop share of mun which # intersects with MVGD in relation to total pop of mun mvgd_pop_per_mun_in_mvgd = ( - mvgd_pop_per_mun_in_mvgd.groupby(level=0) + mvgd_pop_per_mun_in_mvgd.groupby(level=0, as_index=False) .apply(lambda x: x / float(x.sum())) .reset_index() .rename(columns={"pop": "pop_share_mun_in_mvgd"}) @@ -424,7 +424,7 @@ def allocate_evs_numbers(): kba_data = read_kba_data() rs7_data = read_rs7_data() - for scenario_name in ["eGon2035", "eGon100RE"]: + for scenario_name in config.settings()["egon-data"]["--scenarios"]: # Load scenario params scenario_parameters = get_sector_parameters( "mobility", scenario=scenario_name @@ -437,9 +437,7 @@ def allocate_evs_numbers(): scenario_variation_name, scenario_variation_parameters, ) in scenario_parameters.items(): - print(f" SCENARIO VARIATION: {scenario_variation_name}") - # Get EV target ev_target = scenario_variation_parameters["ev_count"] @@ -542,7 +540,7 @@ def get_random_evs(row): .ev_id.to_list() ) - for scenario_name in ["eGon2035", "eGon100RE"]: + for scenario_name in config.settings()["egon-data"]["--scenarios"]: print(f"SCENARIO: {scenario_name}") # Load EVs per grid district @@ -634,7 +632,7 @@ def get_random_evs(row): np.testing.assert_allclose( int(ev_actual), ev_target, - rtol=0.0001, + rtol=0.001, err_msg=f"Dataset on EV numbers allocated to MVGDs " f"seems to be flawed. " f"Scenario: [{scn}], " diff --git a/src/egon/data/datasets/emobility/motorized_individual_travel/helpers.py b/src/egon/data/datasets/emobility/motorized_individual_travel/helpers.py index 7f9e86583..898a30ebf 100644 --- a/src/egon/data/datasets/emobility/motorized_individual_travel/helpers.py +++ b/src/egon/data/datasets/emobility/motorized_individual_travel/helpers.py @@ -82,7 +82,7 @@ "drive_end_timesteps": "drive_end", "consumption_kWh": "consumption", } -MVGD_MIN_COUNT = 3700 if TESTMODE_OFF else 150 +MVGD_MIN_COUNT = 3600 if TESTMODE_OFF else 150 def read_kba_data(): diff --git a/src/egon/data/datasets/emobility/motorized_individual_travel/model_timeseries.py b/src/egon/data/datasets/emobility/motorized_individual_travel/model_timeseries.py index 3496a4a9e..c8372c717 100644 --- a/src/egon/data/datasets/emobility/motorized_individual_travel/model_timeseries.py +++ b/src/egon/data/datasets/emobility/motorized_individual_travel/model_timeseries.py @@ -734,27 +734,38 @@ def write_load( # * load timeseries: # * regular (flex): use driving load # * lowflex: use dumb charging load - if write_lowflex_model is False: - emob_bus_id = write_bus(scenario_name=scenario_name) - write_link(scenario_name=scenario_name) - write_store(scenario_name=scenario_name) + # * status2019: also dumb charging + # * status2023: also dumb charging + + if scenario_name in ["status2019", "status2023"]: write_load( scenario_name=scenario_name, - connection_bus_id=emob_bus_id, - load_ts=( - hourly_load_time_series_df.driving_load_time_series.to_list() # noqa: E501 - ), - ) - else: - # Get lowflex scenario name - lowflex_scenario_name = DATASET_CFG["scenario"]["lowflex"][ - "names" - ][scenario_name] - write_load( - scenario_name=lowflex_scenario_name, connection_bus_id=etrago_bus.bus_id, load_ts=hourly_load_time_series_df.load_time_series.to_list(), - ) + ) + else: + if write_lowflex_model is False: + emob_bus_id = write_bus(scenario_name=scenario_name) + write_link(scenario_name=scenario_name) + write_store(scenario_name=scenario_name) + write_load( + scenario_name=scenario_name, + connection_bus_id=emob_bus_id, + load_ts=( + hourly_load_time_series_df.driving_load_time_series.to_list() # noqa: E501 + ), + ) + + else: + # Get lowflex scenario name + lowflex_scenario_name = DATASET_CFG["scenario"]["lowflex"][ + "names" + ][scenario_name] + write_load( + scenario_name=lowflex_scenario_name, + connection_bus_id=etrago_bus.bus_id, + load_ts=hourly_load_time_series_df.load_time_series.to_list(), + ) def write_to_file(): """Write model data to file (for debugging purposes)""" @@ -1058,6 +1069,23 @@ def generate_model_data_bunch(scenario_name: str, bunch: range) -> None: bat_cap=meta_tech_data.battery_capacity, ) +def generate_model_data_status2019_remaining(): + """Generates timeseries for status2019 scenario for grid districts which + has not been processed in the parallel tasks before. + """ + generate_model_data_bunch( + scenario_name="status2019", + bunch=range(MVGD_MIN_COUNT, len(load_grid_district_ids())), + ) + +def generate_model_data_status2023_remaining(): + """Generates timeseries for status2023 scenario for grid districts which + has not been processed in the parallel tasks before. + """ + generate_model_data_bunch( + scenario_name="status2023", + bunch=range(MVGD_MIN_COUNT, len(load_grid_district_ids())), + ) def generate_model_data_eGon2035_remaining(): """Generates timeseries for eGon2035 scenario for grid districts which diff --git a/src/egon/data/datasets/emobility/motorized_individual_travel/tests.py b/src/egon/data/datasets/emobility/motorized_individual_travel/tests.py index 722135798..649aeaeda 100644 --- a/src/egon/data/datasets/emobility/motorized_individual_travel/tests.py +++ b/src/egon/data/datasets/emobility/motorized_individual_travel/tests.py @@ -30,6 +30,6 @@ def validate_electric_vehicles_numbers(dataset_name, ev_data, ev_target): assert_allclose( ev_data[[_ for _ in CONFIG_EV.keys()]].sum().sum(), ev_target, - rtol=0.0001, + rtol=0.001, err_msg=f"Dataset on EV numbers [{dataset_name}] seems to be flawed.", ) diff --git a/src/egon/data/datasets/emobility/motorized_individual_travel_charging_infrastructure/__init__.py b/src/egon/data/datasets/emobility/motorized_individual_travel_charging_infrastructure/__init__.py index 96a16e042..ae8532b82 100644 --- a/src/egon/data/datasets/emobility/motorized_individual_travel_charging_infrastructure/__init__.py +++ b/src/egon/data/datasets/emobility/motorized_individual_travel_charging_infrastructure/__init__.py @@ -17,6 +17,7 @@ from egon.data.datasets import Dataset from egon.data.datasets.emobility.motorized_individual_travel_charging_infrastructure.db_classes import ( # noqa: E501 EgonEmobChargingInfrastructure, + add_metadata, ) from egon.data.datasets.emobility.motorized_individual_travel_charging_infrastructure.infrastructure_allocation import ( # noqa: E501 run_tracbev, @@ -148,5 +149,6 @@ def __init__(self, dependencies): get_tracbev_data, }, run_tracbev, + add_metadata, ), ) diff --git a/src/egon/data/datasets/emobility/motorized_individual_travel_charging_infrastructure/db_classes.py b/src/egon/data/datasets/emobility/motorized_individual_travel_charging_infrastructure/db_classes.py index 6f4606353..860f1a513 100644 --- a/src/egon/data/datasets/emobility/motorized_individual_travel_charging_infrastructure/db_classes.py +++ b/src/egon/data/datasets/emobility/motorized_individual_travel_charging_infrastructure/db_classes.py @@ -2,11 +2,23 @@ DB tables / SQLAlchemy ORM classes for charging infrastructure """ +import datetime +import json + from geoalchemy2 import Geometry +from omi.dialects import get_dialect from sqlalchemy import Column, Float, Integer, String from sqlalchemy.ext.declarative import declarative_base -from egon.data import config +from egon.data import config, db +from egon.data.metadata import ( + context, + contributors, + generate_resource_fields_from_db_table, + license_odbl, + meta_metadata, + meta_metadata, +) Base = declarative_base() DATASET_CFG = config.datasets()["charging_infrastructure"] @@ -31,3 +43,121 @@ class EgonEmobChargingInfrastructure(Base): srid=DATASET_CFG["original_data"]["sources"]["tracbev"]["srid"] ) ) + + +def add_metadata(): + """ + Add metadata to table grid.egon_emob_charging_infrastructure + """ + contris = contributors(["kh", "kh"]) + + contris[0]["date"] = "2023-03-14" + + contris[0]["object"] = "metadata" + contris[1]["object"] = "dataset" + + contris[0]["comment"] = "Add metadata to dataset." + contris[1]["comment"] = "Add workflow to generate dataset." + + meta = { + "name": "grid.egon_emob_charging_infrastructure", + "title": "eGon Electromobility Charging Infrastructure", + "id": "WILL_BE_SET_AT_PUBLICATION", + "description": ( + "Identified sites for charging infrastructure for motorized " + "individual travel using TracBEV" + ), + "language": "en-US", + "keywords": [ + "mit", + "charging", + "infrastructure", + "electromobility", + "tracbev", + ], + "publicationDate": datetime.date.today().isoformat(), + "context": context(), + "spatial": { + "location": None, + "extent": "Germany", + "resolution": "1 m", + }, + "temporal": { + "referenceDate": "2022-04-21", + "timeseries": {}, + }, + "sources": [ + { + "title": "TracBEV input data", + "description": ( + "This data set is used with the software tool TracBEV to " + "calculate locations for charging infrastructure from " + "SimBEV results." + ), + "path": "https://zenodo.org/record/6466480#.YmE9xtPP1hE", + "licenses": [license_odbl(attribution="© Schiel, Moritz")], + } + ], + "licenses": [ + license_odbl(attribution="© eGon development team"), + ], + "contributors": contris, + "resources": [ + { + "profile": "tabular-data-resource", + "name": "grid.egon_emob_charging_infrastructure", + "path": "None", + "format": "PostgreSQL", + "encoding": "UTF-8", + "schema": { + "fields": generate_resource_fields_from_db_table( + DATASET_CFG["targets"]["charging_infrastructure"][ + "schema" + ], + DATASET_CFG["targets"]["charging_infrastructure"][ + "table" + ], + ), + "primaryKey": "cp_id", + }, + "dialect": {"delimiter": "", "decimalSeparator": ""}, + } + ], + "review": {"path": "", "badge": ""}, + "metaMetadata": meta_metadata(), + "_comment": { + "metadata": ( + "Metadata documentation and explanation (https://github.com/Op" + "enEnergyPlatform/oemetadata/blob/master/metadata/v141/metadat" + "a_key_description.md)" + ), + "dates": ( + "Dates and time must follow the ISO8601 including time zone " + "(YYYY-MM-DD or YYYY-MM-DDThh:mm:ss±hh)" + ), + "units": "Use a space between numbers and units (100 m)", + "languages": ( + "Languages must follow the IETF (BCP47) format (en-GB, en-US, " + "de-DE)" + ), + "licenses": ( + "License name must follow the SPDX License List " + "(https://spdx.org/licenses/)" + ), + "review": ( + "Following the OEP Data Review (https://github.com/OpenEnergyP" + "latform/data-preprocessing/wiki)" + ), + "none": "If not applicable use (none)", + }, + } + + dialect = get_dialect(f"oep-v{meta_metadata()['metadataVersion'][4:7]}")() + + meta = dialect.compile_and_render(dialect.parse(json.dumps(meta))) + + db.submit_comment( + f"'{json.dumps(meta)}'", + DATASET_CFG["targets"]["charging_infrastructure"]["schema"], + DATASET_CFG["targets"]["charging_infrastructure"]["table"], + ) diff --git a/src/egon/data/datasets/emobility/motorized_individual_travel_charging_infrastructure/use_cases.py b/src/egon/data/datasets/emobility/motorized_individual_travel_charging_infrastructure/use_cases.py index e0081d90f..18a20e888 100644 --- a/src/egon/data/datasets/emobility/motorized_individual_travel_charging_infrastructure/use_cases.py +++ b/src/egon/data/datasets/emobility/motorized_individual_travel_charging_infrastructure/use_cases.py @@ -151,7 +151,6 @@ def distribute_by_poi(region_poi: gpd.GeoDataFrame, num_points: int | float): def match_existing_points( region_points: gpd.GeoDataFrame, region_poi: gpd.GeoDataFrame ): - region_poi = region_poi.assign(exists=False) poi_buffer = region_poi.buffer(region_poi["radius"].astype(int)) region_points = region_points.assign(potential=0) @@ -165,7 +164,7 @@ def match_existing_points( # decent average as fallback region_points.at[i, "potential"] = 5 elif num_clusters == 1: - region_points.at[i, "potential"] = clusters["potential"] + region_points.at[i, "potential"] = clusters["potential"].values[0] region_poi.loc[cluster, "exists"] = True elif num_clusters > 1: diff --git a/src/egon/data/datasets/era5.py b/src/egon/data/datasets/era5.py index ffd6085fe..446d19d53 100644 --- a/src/egon/data/datasets/era5.py +++ b/src/egon/data/datasets/era5.py @@ -92,7 +92,8 @@ def import_cutout(boundary="Europe"): Weather data stored in cutout """ - weather_year = get_sector_parameters("global", "eGon2035")["weather_year"] + weather_year = get_sector_parameters("global", "status2023")["weather_year"] + # This is fixed to one scenario as its currently not possible to have multiple weather-years if boundary == "Europe": xs = slice(-12.0, 35.1) @@ -195,7 +196,7 @@ def insert_weather_cells(): cutout = import_cutout() df = gpd.GeoDataFrame( - {"geom": cutout.grid_cells()}, geometry="geom", crs=4326 + {"geom": cutout.grid.geometry}, geometry="geom", crs=4326 ) df.to_postgis( diff --git a/src/egon/data/datasets/etrago_setup.py b/src/egon/data/datasets/etrago_setup.py index 6d722af65..d3f4975cc 100755 --- a/src/egon/data/datasets/etrago_setup.py +++ b/src/egon/data/datasets/etrago_setup.py @@ -1,4 +1,7 @@ # coding: utf-8 +import datetime +import json + from geoalchemy2.types import Geometry from shapely.geometry import LineString from sqlalchemy import ( @@ -17,27 +20,159 @@ from sqlalchemy.ext.declarative import declarative_base import geopandas as gpd import pandas as pd +import pypsa from egon.data import db from egon.data.datasets import Dataset +from egon.data.metadata import ( + context, + contributors, + license_egon_data_odbl, + meta_metadata, + sources +) Base = declarative_base() metadata = Base.metadata +network = pypsa.Network() +# add Storage key (called StorageUnit in PyPSA) +network.component_attrs["Storage"] = network.component_attrs["StorageUnit"] + + +def get_pypsa_field_descriptors(component, timeseries=False): + + ident = component.lower() + "_id" + + data = network.component_attrs[component].rename({"name": ident}) + data = data[data.status != "Output"] + + if timeseries: + data = data[data["type"].str.contains("series")] + data.loc["temp_id"] = [ + "integer", + "n/a", + "n/a", + "Unique identifyier of temporal index", + "Input", + ] + + data.loc[ident, "type"] = "int" + data.loc["scn_name"] = [ + "string", + "n/a", + "n/a", + "Name of the eGon scenario", + "Input", + ] + data.unit.fillna("none", inplace=True) + data.default.fillna("n/a", inplace=True) + return data + + +def get_meta( + schema, + component, + description="TODO", + source_list=[], + contributor_list=[], + timeseries=False, +): + + table = "egon_etrago_" + component.lower() + + if timeseries: + table = table + "_timeseries" + fields = ( + get_pypsa_field_descriptors(component, timeseries) + .reset_index() + .to_dict(orient="records") + ) + # geometry column still missing + + meta = { + "name": schema + "." + table, + "title": component, + "id": "WILL_BE_SET_AT_PUBLICATION", + # no automatic description? PyPSA descriptions do not quite fit our + # scope + "description": description, + "language": ["en-EN"], + "publicationDate": datetime.date.today().isoformat(), + "context": context(), + "spatial": { + "location": None, + "extent": "Germany", + "resolution": None, + }, + "sources": source_list, + "licenses": [license_egon_data_odbl()], + "contributors": contributor_list, + "resources": [ + { + "profile": "tabular-data-resource", + "name": schema + "." + table, + "path": None, + "format": "PostgreSQL", + "encoding": "UTF-8", + "schema": { + "fields": fields, + "primaryKey": ["scn_name", component.lower() + "_id"], + "foreignKeys": [], + }, + "dialect": { + "delimiter": "", + "decimalSeparator": "" + }, + } + ], + "metaMetadata": meta_metadata(), + } + + # Create json dump + meta_json = "'" + json.dumps(meta, indent=4, ensure_ascii=False) + "'" + + return meta_json + class EtragoSetup(Dataset): def __init__(self, dependencies): super().__init__( name="EtragoSetup", - version="0.0.10", + version="0.0.11", dependencies=dependencies, tasks=(create_tables, {temp_resolution, insert_carriers}), ) class EgonPfHvBus(Base): + + source_list = [ + sources()["egon-data"], + sources()["openstreetmap"], + sources()["peta"], + sources()["SciGRID_gas"], + sources()["bgr_inspeeds_data_bundle"], + ] + + contributor_list = contributors(["ic", "cb", "ke", "an", "fw"]) + contributor_list[0]["comment"] = "Added electricity substations" + contributor_list[1]["comment"] = "Added heat buses" + contributor_list[2]["comment"] = "Added DSM buses" + contributor_list[3]["comment"] = "Added CH4 sector buses" + contributor_list[4]["comment"] = "Added H2 sector buses" + + __tablename__ = "egon_etrago_bus" - __table_args__ = {"schema": "grid"} + __table_args__ = { + "schema": "grid", + "comment": get_meta( + "grid", + "Bus", + source_list=source_list, + contributor_list=contributor_list, + ), + } scn_name = Column(String, primary_key=True, nullable=False) bus_id = Column(BigInteger, primary_key=True, nullable=False) @@ -54,8 +189,25 @@ class EgonPfHvBus(Base): class EgonPfHvBusTimeseries(Base): + + source_list = [ + sources()["egon-data"], + ] + + contributor_list = contributors(["cb"]) + contributor_list[0]["comment"] = "Added metadata" + __tablename__ = "egon_etrago_bus_timeseries" - __table_args__ = {"schema": "grid"} + __table_args__ = { + "schema": "grid", + "comment": get_meta( + "grid", + "Bus", + source_list=source_list, + contributor_list=contributor_list, + timeseries=True, + ), + } scn_name = Column(String, primary_key=True, nullable=False) bus_id = Column(BigInteger, primary_key=True, nullable=False) @@ -63,8 +215,37 @@ class EgonPfHvBusTimeseries(Base): class EgonPfHvGenerator(Base): + + source_list = [ + sources()["egon-data"], + sources()["openstreetmap"], + sources()["mastr"], + sources()["nep2021"], + sources()["tyndp"], + sources()["SciGRID_gas"], + sources()["Einspeiseatlas"], + sources()["technology-data"], + sources()["vg250"], + ] + + contributor_list = contributors(["ic", "cb", "ce", "an", "ke"]) + contributor_list[0]["comment"] = "Added hydro and biomass plants" + contributor_list[1]["comment"] = "Added solar and geothermal plants" + contributor_list[2]["comment"] = "Added wind on- and offshore plants" + contributor_list[3]["comment"] = "Added gas feedin generators" + contributor_list[4]["comment"] = "Added pv ground mounted" + + __tablename__ = "egon_etrago_generator" - __table_args__ = {"schema": "grid"} + __table_args__ = { + "schema": "grid", + "comment": get_meta( + "grid", + "Generator", + source_list=source_list, + contributor_list=contributor_list, + ), + } scn_name = Column(String, primary_key=True, nullable=False) generator_id = Column(BigInteger, primary_key=True, nullable=False) @@ -103,8 +284,29 @@ class EgonPfHvGenerator(Base): class EgonPfHvGeneratorTimeseries(Base): + + source_list = [ + sources()["egon-data"], + sources()["era5"], + ] + + contributor_list = contributors(["cb"]) + contributor_list[0][ + "comment" + ] = "Added p_max_pu timeseries for pv and wind" + + __tablename__ = "egon_etrago_generator_timeseries" - __table_args__ = {"schema": "grid"} + __table_args__ = { + "schema": "grid", + "comment": get_meta( + "grid", + "Generator", + source_list=source_list, + contributor_list=contributor_list, + timeseries=True, + ), + } scn_name = Column(String, primary_key=True, nullable=False) generator_id = Column(Integer, primary_key=True, nullable=False) @@ -117,8 +319,26 @@ class EgonPfHvGeneratorTimeseries(Base): class EgonPfHvLine(Base): + + source_list = [ + sources()["egon-data"], + sources()["openstreetmap"], + ] + + contributor_list = contributors(["ic", "cb"]) + contributor_list[0]["comment"] = "Added lines from osmTGmod tables" + contributor_list[1]["comment"] = "Added meta data" + __tablename__ = "egon_etrago_line" - __table_args__ = {"schema": "grid"} + __table_args__ = { + "schema": "grid", + "comment": get_meta( + "grid", + "Line", + source_list=source_list, + contributor_list=contributor_list, + ), + } scn_name = Column(String, primary_key=True, nullable=False) line_id = Column(BigInteger, primary_key=True, nullable=False) @@ -150,8 +370,28 @@ class EgonPfHvLine(Base): class EgonPfHvLineTimeseries(Base): + + source_list = [ + sources()["egon-data"], + sources()["nep2021"], + sources()["era5"], + ] + + contributor_list = contributors(["ce", "cb"]) + contributor_list[0]["comment"] = "Added s_max_pu timeseries" + contributor_list[1]["comment"] = "Added meta data" + __tablename__ = "egon_etrago_line_timeseries" - __table_args__ = {"schema": "grid"} + __table_args__ = { + "schema": "grid", + "comment": get_meta( + "grid", + "Line", + source_list=source_list, + contributor_list=contributor_list, + timeseries=True, + ), + } scn_name = Column(String, primary_key=True, nullable=False) line_id = Column(BigInteger, primary_key=True, nullable=False) @@ -160,8 +400,44 @@ class EgonPfHvLineTimeseries(Base): class EgonPfHvLink(Base): + + source_list = [ + sources()["egon-data"], + sources()["openstreetmap"], + sources()["nep2021"], + sources()["peta"], + sources()["mastr"], + sources()["SciGRID_gas"], + sources()["pipeline_classification"], + sources()["technology-data"], + sources()["dsm-heitkoetter"], + sources()["schmidt"], + sources()["hotmaps_industrial_sites"], + sources()["demandregio"], + ] + + contributor_list = contributors(["ic", "cb", "ke", "ja", "fw", "an"]) + contributor_list[0]["comment"] = "Added DC lines from osmTGmod tables" + contributor_list[1]["comment"] = "Added CHPs and heat links" + contributor_list[2]["comment"] = "Added DSM links" + contributor_list[3]["comment"] = "Added e-Mobility links" + contributor_list[4]["comment"] = "Added H2 related links" + contributor_list[5]["comment"] = "Added CH4 links" + + __tablename__ = "egon_etrago_link" - __table_args__ = {"schema": "grid"} + __table_args__ = { + "schema": "grid", + "comment": get_meta( + "grid", + "Link", + source_list=source_list, + contributor_list=contributor_list, + ), + } + + __tablename__ = "egon_etrago_link" + __table_args__ = {"schema": "grid", "comment": get_meta("grid", "Link")} scn_name = Column(String, primary_key=True, nullable=False) link_id = Column(BigInteger, primary_key=True, nullable=False) @@ -188,8 +464,34 @@ class EgonPfHvLink(Base): class EgonPfHvLinkTimeseries(Base): + source_list = [ + sources()["egon-data"], + sources()["era5"], + sources()["dsm-heitkoetter"], + sources()["schmidt"], + sources()["hotmaps_industrial_sites"], + sources()["openstreetmap"], + sources()["demandregio"], + ] + + contributor_list = contributors(["cb", "ke", "ja"]) + contributor_list[0][ + "comment" + ] = "Added efficiency timeseries for heat pumps" + contributor_list[1]["comment"] = "Added dsm link timeseries" + contributor_list[2]["comment"] = "Added e mobility link timeseries" + __tablename__ = "egon_etrago_link_timeseries" - __table_args__ = {"schema": "grid"} + __table_args__ = { + "schema": "grid", + "comment": get_meta( + "grid", + "Link", + source_list=source_list, + contributor_list=contributor_list, + timeseries=True, + ), + } scn_name = Column(String, primary_key=True, nullable=False) link_id = Column(BigInteger, primary_key=True, nullable=False) @@ -202,8 +504,36 @@ class EgonPfHvLinkTimeseries(Base): class EgonPfHvLoad(Base): + + source_list = [ + sources()["egon-data"], + sources()["demandregio"], + sources()["nep2021"], + sources()["peta"], + sources()["schmidt"], + sources()["hotmaps_industrial_sites"], + sources()["openstreetmap"], + sources()["openffe_gas"], + sources()["tyndp"], + ] + + contributor_list = contributors(["ic", "cb", "an", "ja"]) + contributor_list[0]["comment"] = "Added electrical demands" + contributor_list[1]["comment"] = "Added heat deands" + contributor_list[2]["comment"] = "Added gas demands" + contributor_list[3]["comment"] = "Added mobility demands" + + __tablename__ = "egon_etrago_load" - __table_args__ = {"schema": "grid"} + __table_args__ = { + "schema": "grid", + "comment": get_meta( + "grid", + "Load", + source_list=source_list, + contributor_list=contributor_list, + ), + } scn_name = Column(String, primary_key=True, nullable=False) load_id = Column(BigInteger, primary_key=True, nullable=False) @@ -216,8 +546,37 @@ class EgonPfHvLoad(Base): class EgonPfHvLoadTimeseries(Base): + source_list = [ + sources()["egon-data"], + sources()["demandregio"], + sources()["nep2021"], + sources()["peta"], + sources()["openffe_gas"], + sources()["tyndp"], + sources()["era5"], + sources()["schmidt"], + sources()["hotmaps_industrial_sites"], + sources()["openstreetmap"], + ] + + contributor_list = contributors(["cb", "ic", "ja", "an"]) + contributor_list[0]["comment"] = "Added heat load timeseries" + contributor_list[1]["comment"] = "Added electricity load timeseries" + contributor_list[2]["comment"] = "Added e mobility load timeseries" + contributor_list[3]["comment"] = "Added gas load timeseries" + + __tablename__ = "egon_etrago_load_timeseries" - __table_args__ = {"schema": "grid"} + __table_args__ = { + "schema": "grid", + "comment": get_meta( + "grid", + "Load", + source_list=source_list, + contributor_list=contributor_list, + timeseries=True, + ), + } scn_name = Column(String, primary_key=True, nullable=False) load_id = Column(BigInteger, primary_key=True, nullable=False) @@ -227,8 +586,24 @@ class EgonPfHvLoadTimeseries(Base): class EgonPfHvCarrier(Base): + source_list = [ + sources()["egon-data"], + ] + + contributor_list = contributors(["fw"]) + contributor_list[0]["comment"] = "Added list of carriers" + + __tablename__ = "egon_etrago_carrier" - __table_args__ = {"schema": "grid"} + __table_args__ = { + "schema": "grid", + "comment": get_meta( + "grid", + "Carrier", + source_list=source_list, + contributor_list=contributor_list, + ), + } name = Column(Text, primary_key=True, nullable=False) co2_emissions = Column(Float(53), server_default="0.") @@ -238,8 +613,30 @@ class EgonPfHvCarrier(Base): class EgonPfHvStorage(Base): + + source_list = [ + sources()["egon-data"], + sources()["nep2021"], + sources()["mastr"], + sources()["technology-data"], + ] + + contributor_list = contributors(["ic"]) + contributor_list[0][ + "comment" + ] = "Added battery and pumped hydro storage units" + + __tablename__ = "egon_etrago_storage" - __table_args__ = {"schema": "grid"} + __table_args__ = { + "schema": "grid", + "comment": get_meta( + "grid", + "Storage", + source_list=source_list, + contributor_list=contributor_list, + ), + } scn_name = Column(String, primary_key=True, nullable=False) storage_id = Column(BigInteger, primary_key=True, nullable=False) @@ -271,8 +668,25 @@ class EgonPfHvStorage(Base): class EgonPfHvStorageTimeseries(Base): + source_list = [ + sources()["egon-data"], + ] + + contributor_list = contributors(["cb"]) + contributor_list[0]["comment"] = "Added metadata" + + __tablename__ = "egon_etrago_storage_timeseries" - __table_args__ = {"schema": "grid"} + __table_args__ = { + "schema": "grid", + "comment": get_meta( + "grid", + "Storage", + source_list=source_list, + contributor_list=contributor_list, + timeseries=True, + ), + } scn_name = Column(String, primary_key=True, nullable=False) storage_id = Column(BigInteger, primary_key=True, nullable=False) @@ -287,8 +701,38 @@ class EgonPfHvStorageTimeseries(Base): class EgonPfHvStore(Base): + source_dict = sources() + + source_list = [ + source_dict["bgr_inspee"], + source_dict["bgr_inspeeds"], + source_dict["bgr_inspeeds_data_bundle"], + source_dict["bgr_inspeeds_report"], + source_dict["SciGRID_gas"], + sources()["technology-data"], + sources()["dsm-heitkoetter"], + sources()["schmidt"], + sources()["hotmaps_industrial_sites"], + sources()["openstreetmap"], + sources()["demandregio"], + ] + contributor_list = contributors(["an", "fw", "ke", "cb", "ja"]) + contributor_list[0]["comment"] = "Add H2 storage" + contributor_list[1]["comment"] = "Add CH4 storage" + contributor_list[2]["comment"] = "Add DSM storage" + contributor_list[3]["comment"] = "Add heat storage" + contributor_list[4]["comment"] = "Add e-mobility storage" + __tablename__ = "egon_etrago_store" - __table_args__ = {"schema": "grid"} + __table_args__ = { + "schema": "grid", + "comment": get_meta( + "grid", + "Store", + source_list=source_list, + contributor_list=contributor_list, + ), + } scn_name = Column(String, primary_key=True, nullable=False) store_id = Column(BigInteger, primary_key=True, nullable=False) @@ -314,9 +758,31 @@ class EgonPfHvStore(Base): class EgonPfHvStoreTimeseries(Base): - __tablename__ = "egon_etrago_store_timeseries" - __table_args__ = {"schema": "grid"} + source_dict = sources() + # TODO: Add other sources for dsm + source_list = [ + sources()["technology-data"], + sources()["dsm-heitkoetter"], + sources()["schmidt"], + sources()["hotmaps_industrial_sites"], + sources()["openstreetmap"], + sources()["demandregio"], + ] + contributor_list = contributors(["ke", "ja"]) + contributor_list[0]["comment"] = "Add DSM storage" + contributor_list[1]["comment"] = "Add e-mobility storage" + __tablename__ = "egon_etrago_store_timeseries" + __table_args__ = { + "schema": "grid", + "comment": get_meta( + "grid", + "Store", + source_list=source_list, + contributor_list=contributor_list, + timeseries=True, + ), + } scn_name = Column(String, primary_key=True, nullable=False) store_id = Column(BigInteger, primary_key=True, nullable=False) temp_id = Column(Integer, primary_key=True, nullable=False) @@ -338,8 +804,27 @@ class EgonPfHvTempResolution(Base): class EgonPfHvTransformer(Base): + + source_list = [ + sources()["egon-data"], + sources()["openstreetmap"], + ] + + contributor_list = contributors(["ic", "cb"]) + contributor_list[0]["comment"] = "Added transformes from osmTGmod tables" + contributor_list[1]["comment"] = "Added meta data" + + __tablename__ = "egon_etrago_transformer" - __table_args__ = {"schema": "grid"} + __table_args__ = { + "schema": "grid", + "comment": get_meta( + "grid", + "Transformer", + source_list=source_list, + contributor_list=contributor_list, + ), + } scn_name = Column(String, primary_key=True, nullable=False) trafo_id = Column(BigInteger, primary_key=True, nullable=False) @@ -371,8 +856,24 @@ class EgonPfHvTransformer(Base): class EgonPfHvTransformerTimeseries(Base): + source_list = [ + sources()["egon-data"], + ] + + contributor_list = contributors(["cb"]) + contributor_list[0]["comment"] = "Added meta data" + __tablename__ = "egon_etrago_transformer_timeseries" - __table_args__ = {"schema": "grid"} + __table_args__ = { + "schema": "grid", + "comment": get_meta( + "grid", + "Transformer", + source_list=source_list, + contributor_list=contributor_list, + timeseries=True, + ), + } scn_name = Column(String, primary_key=True, nullable=False) trafo_id = Column(BigInteger, primary_key=True, nullable=False) diff --git a/src/egon/data/datasets/fill_etrago_gen.py b/src/egon/data/datasets/fill_etrago_gen.py index cae8fff4f..8900c436e 100644 --- a/src/egon/data/datasets/fill_etrago_gen.py +++ b/src/egon/data/datasets/fill_etrago_gen.py @@ -126,18 +126,19 @@ def add_marginal_costs(power_plants): for carrier in pp_scn.carrier.unique(): if carrier not in (marginal_costs.index): warning.append(carrier) - marginal_costs = marginal_costs.append( - pd.Series(name=carrier, data={"marginal_cost": 0}) - ) + marginal_costs.at[carrier, "marginal_cost"] = 0 if warning: print( - f"""There are not marginal_cost values for: \n{warning} + f"""There are no marginal_cost values for: \n{warning} in the scenario {scenario}. Missing values set to 0""" ) - pp = pp.append( - pp_scn.merge( - right=marginal_costs, left_on="carrier", right_index=True - ) + pp = pd.concat( + [ + pp, + pp_scn.merge( + right=marginal_costs, left_on="carrier", right_index=True + ), + ] ) return pp @@ -174,6 +175,7 @@ def fill_etrago_gen_time_table( etrago_pp_time = etrago_pp_time[ (etrago_pp_time["carrier"] == "solar") + | (etrago_pp_time["carrier"] == "solar_rooftop") | (etrago_pp_time["carrier"] == "wind_onshore") | (etrago_pp_time["carrier"] == "wind_offshore") ] @@ -255,10 +257,10 @@ def numpy_nan(data): def power_timeser(weather_data): - if len(set(weather_data)) <= 1: - return weather_data.iloc[0] - else: + if weather_data.isna().any(): return -1 + else: + return weather_data.iloc[0] def adjust_renew_feedin_table(renew_feedin, cfg): @@ -266,6 +268,11 @@ def adjust_renew_feedin_table(renew_feedin, cfg): carrier_pv_mask = renew_feedin["carrier"] == "pv" renew_feedin.loc[carrier_pv_mask, "carrier"] = "solar" + # Copy solar timeseries for solar_rooftop + feedin_solar_rooftop = renew_feedin.loc[renew_feedin["carrier"]=="solar"] + feedin_solar_rooftop.loc[:, "carrier"] = "solar_rooftop" + renew_feedin = pd.concat([renew_feedin, feedin_solar_rooftop], ignore_index=True) + # convert renewable feedin lists to arrays renew_feedin["feedin"] = renew_feedin["feedin"].apply(np.array) @@ -302,30 +309,56 @@ def delete_previuos_gen(cfg, con, etrago_gen_orig, power_plants): def set_timeseries(power_plants, renew_feedin): + """ + Create a function to calculate the feed-in timeseries for power plants. + + Parameters + ---------- + power_plants : DataFrame + A DataFrame containing information about power plants, including their bus IDs, + carriers, weather cell IDs, and electrical capacities. + renew_feedin : DataFrame + A DataFrame containing feed-in values for different carriers and weather cell IDs. + + Returns + ------- + function + A function that takes a power plant object and returns its feed-in value based on + either its direct weather cell ID or the aggregated feed-in of all power plants + connected to the same bus and having the same carrier. + """ + def timeseries(pp): + """Calculate the feed-in for a given power plant based on weather cell ID or aggregation.""" if pp.weather_cell_id != -1: - feedin_time = renew_feedin[ + # Directly fetch feed-in value for power plants with an associated weather cell ID + return renew_feedin.loc[ (renew_feedin["w_id"] == pp.weather_cell_id) - & (renew_feedin["carrier"] == pp.carrier) - ].feedin.iloc[0] - return feedin_time + & (renew_feedin["carrier"] == pp.carrier), + "feedin", + ].iat[0] else: - df = power_plants[ + # Aggregate feed-in for power plants without a direct weather cell association + df = power_plants.loc[ (power_plants["bus_id"] == pp.bus_id) & (power_plants["carrier"] == pp.carrier) - ] - total_int_cap = df.el_capacity.sum() - df["feedin"] = 0 + ].dropna(subset=["weather_cell_id"]) + + total_int_cap = df["el_capacity"].sum() + + # Fetch and calculate proportional feed-in for each power plant df["feedin"] = df.apply( - lambda x: renew_feedin[ + lambda x: renew_feedin.loc[ (renew_feedin["w_id"] == x.weather_cell_id) - & (renew_feedin["carrier"] == x.carrier) - ].feedin.iloc[0], + & (renew_feedin["carrier"] == x.carrier), + "feedin", + ].iat[0], axis=1, ) - df["feedin"] = df.apply( - lambda x: x.el_capacity / total_int_cap * x.feedin, axis=1 - ) - return df.feedin.sum() + + # Calculate and return aggregated feed-in based on electrical capacity + return df.apply( + lambda x: x["el_capacity"] / total_int_cap * x["feedin"], axis=1 + ).sum() return timeseries diff --git a/src/egon/data/datasets/fix_ehv_subnetworks.py b/src/egon/data/datasets/fix_ehv_subnetworks.py index 96d780ef6..54f86b56f 100644 --- a/src/egon/data/datasets/fix_ehv_subnetworks.py +++ b/src/egon/data/datasets/fix_ehv_subnetworks.py @@ -4,10 +4,11 @@ import numpy as np import pandas as pd -from egon.data import db +from egon.data import config, db, logger from egon.data.config import settings from egon.data.datasets import Dataset from egon.data.datasets.etrago_setup import link_geom_from_buses +from egon.data.datasets.scenario_parameters import get_sector_parameters class FixEhvSubnetworks(Dataset): @@ -45,7 +46,7 @@ def __init__(self, dependencies): ) -def select_bus_id(x, y, v_nom, scn_name, carrier): +def select_bus_id(x, y, v_nom, scn_name, carrier, find_closest=False): bus_id = db.select_dataframe( f""" SELECT bus_id @@ -59,8 +60,27 @@ def select_bus_id(x, y, v_nom, scn_name, carrier): ) if bus_id.empty: - return None + logger.info("No bus found") + if find_closest: + logger.info(f"Finding closest to x = {x}, y = {y}") + bus_id = db.select_dataframe( + f""" + SELECT bus_id, st_distance(geom, 'SRID=4326;POINT({x} {y})'::geometry) + FROM grid.egon_etrago_bus + WHERE v_nom = {v_nom} + AND scn_name = '{scn_name}' + AND carrier = '{carrier}' + ORDER BY st_distance + Limit 1 + """ + ) + logger.info(f"Bus ID = {bus_id.bus_id[0]} selected") + return bus_id.bus_id[0] + else: + logger.info("Find closest == False.") + return None else: + logger.info(f"Exact match with bus ID = {bus_id.bus_id[0]} found.") return bus_id.bus_id[0] @@ -89,7 +109,7 @@ def add_bus(x, y, v_nom, scn_name): def drop_bus(x, y, v_nom, scn_name): bus = select_bus_id(x, y, v_nom, scn_name, carrier="AC") - if bus != None: + if bus is not None: db.execute_sql( f""" DELETE FROM grid.egon_etrago_bus @@ -103,8 +123,13 @@ def drop_bus(x, y, v_nom, scn_name): def add_line(x0, y0, x1, y1, v_nom, scn_name, cables): - bus0 = select_bus_id(x0, y0, v_nom, scn_name, carrier="AC") - bus1 = select_bus_id(x1, y1, v_nom, scn_name, carrier="AC") + parameters = get_sector_parameters("electricity", scenario=scn_name) + bus0 = select_bus_id( + x0, y0, v_nom, scn_name, carrier="AC", find_closest=True + ) + bus1 = select_bus_id( + x1, y1, v_nom, scn_name, carrier="AC", find_closest=True + ) df = pd.DataFrame( index=[db.next_etrago_id("line")], @@ -122,20 +147,33 @@ def add_line(x0, y0, x1, y1, v_nom, scn_name, cables): gdf["length"] = gdf.to_crs(3035).topo.length.mul(1e-3) + # all the values used for x, r and b are taken from the electrical values + # reference table from oemtgmod: github.com/wupperinst/osmTGmod if v_nom == 220: s_nom = 520 x_per_km = 0.001 * 2 * np.pi * 50 + r_per_km = 0.05475 + b_per_km = 11 * 2 * np.pi * 50 * 1e-9 + cost_per_km = parameters["capital_cost"]["ac_ehv_overhead_line"] elif v_nom == 380: s_nom = 1790 x_per_km = 0.0008 * 2 * np.pi * 50 + r_per_km = 0.027375 + b_per_km = 14 * 2 * np.pi * 50 * 1e-9 + cost_per_km = parameters["capital_cost"]["ac_ehv_overhead_line"] gdf["s_nom"] = s_nom * gdf["cables"] / 3 + gdf["s_nom_extendable"] = True + gdf["lifetime"] = parameters["lifetime"]["ac_ehv_overhead_line"] + gdf["s_nom_min"] = s_nom * gdf["cables"] / 3 gdf["x"] = (x_per_km * gdf["length"]) / (gdf["cables"] / 3) + gdf["r"] = (r_per_km * gdf["length"]) / (gdf["cables"] / 3) + gdf["b"] = (b_per_km * gdf["length"]) * (gdf["cables"] / 3) + gdf["capital_cost"] = (cost_per_km * gdf["length"]) * (gdf["cables"] / 3) gdf.index.name = "line_id" - gdf.reset_index().to_postgis( "egon_etrago_line", schema="grid", con=db.engine(), if_exists="append" ) @@ -145,7 +183,7 @@ def drop_line(x0, y0, x1, y1, v_nom, scn_name): bus0 = select_bus_id(x0, y0, v_nom, scn_name, carrier="AC") bus1 = select_bus_id(x1, y1, v_nom, scn_name, carrier="AC") - if (bus0 != None) and (bus1 != None): + if (bus0 is not None) and (bus1 is not None): db.execute_sql( f""" DELETE FROM grid.egon_etrago_line @@ -159,8 +197,12 @@ def drop_line(x0, y0, x1, y1, v_nom, scn_name): def add_trafo(x, y, v_nom0, v_nom1, scn_name, n=1): - bus0 = select_bus_id(x, y, v_nom0, scn_name, carrier="AC") - bus1 = select_bus_id(x, y, v_nom1, scn_name, carrier="AC") + bus0 = select_bus_id( + x, y, v_nom0, scn_name, carrier="AC", find_closest=True + ) + bus1 = select_bus_id( + x, y, v_nom1, scn_name, carrier="AC", find_closest=True + ) df = pd.DataFrame( index=[db.next_etrago_id("line")], @@ -195,7 +237,7 @@ def drop_trafo(x, y, v_nom0, v_nom1, scn_name): bus0 = select_bus_id(x, y, v_nom0, scn_name, carrier="AC") bus1 = select_bus_id(x, y, v_nom1, scn_name, carrier="AC") - if (bus0 != None) and (bus1 != None): + if (bus0 is not None) and (bus1 is not None): db.execute_sql( f""" DELETE FROM grid.egon_etrago_transformer @@ -218,26 +260,13 @@ def fix_subnetworks(scn_name): add_line( 10.760835327266625, # Lübeck Siems 53.90974536547805, - 10.640952461335745, # Umspannwerk Lübeck - 53.91944427801032, + 10.641467804496818, # Umspannwerk Lübeck + 53.91916269128779, 220, scn_name, 3, ) - # Missing 220kV line from Audorf to Kiel - add_line( - # Audorf - 9.726992766257577, - 54.291420962253234, - # Kiel - 9.9572075, - 54.321589, - 220, - scn_name, - 6, - ) - if settings()["egon-data"]["--dataset-boundary"] == "Everything": # Missing line from USW Uchtelfangen to 'Kraftwerk Weiher' add_line( @@ -249,151 +278,35 @@ def fix_subnetworks(scn_name): scn_name, 6, ) - - # Missing 380kV line near Elsfleth - add_line( - # Line - 8.419326700000001, - 53.229867000000006, - # Schaltanlage Elsfleth/West - 8.402976949446648, - 53.2371468322213, - 380, - scn_name, - 6, - ) - - # Missing 380kV line near Magdala + if ( + select_bus_id( + 12.85381530378627, 48.764209444817745, 380, scn_name, "AC" + ) + != None + ): + # Missing line from Umspannwerk Plottling to Gänsdorf UW + add_line( + 12.85381530378627, # Umspannwerk Plottling + 48.764209444817745, + 12.769768646403532, # Gänsdorf UW + 48.80533685376445, + 380, + scn_name, + 3, + ) + + # Missing line inside Ottstedt add_line( - # Line north south - 11.4298432, - 50.9117467, - # Line east - 11.4295305, + 11.4295305, # Ottstedt 50.9115176, + 11.4299277, # Ottstedt + 50.911449600000005, 380, scn_name, 3, ) - # Missing 220kV line near Frimmersdorf - add_line( - # Line west - 6.585418000000001, - 51.0495723, - # Line east - 6.5867616, - 51.0520915, - 220, - scn_name, - 6, - ) - - # Missing 220kV line from Wolmirstedt to Stendal - add_line( - # Wolmirstedt - 11.637225336209951, - 52.26707328151311, - # Stendal - 11.7689, - 52.505533, - 220, - scn_name, - 6, - ) - - # Plattling - # Update way for osmTGmod in - # 'LINESTRING (12.85328076018362 48.76616932172957, - # 12.85221826521118 48.76597882857125, - # 12.85092755963579 48.76451816626182, - # 12.85081583430311 48.76336597271223, - # 12.85089191559093 48.76309793961921, - # 12.85171674549663 48.76313124988151, - # 12.85233496021983 48.76290980724934, - # 12.85257485139349 48.76326650768988, - # 12.85238077788078 48.76354965879587, - # 12.85335698387775 48.76399030383004, - # 12.85444925633996 48.76422235417385, - # 12.853289544662 48.76616304929393)' - - # Lamspringe 380kV lines - drop_line( - 9.988215035677026, - 51.954230057487926, - 9.991477300000001, - 51.939711, - 380, - scn_name, - ) - - drop_line( - 9.995589, - 51.969716000000005, - 9.988215035677026, - 51.954230057487926, - 380, - scn_name, - ) - - drop_line( - 9.982829, - 51.985980000000005, - 9.995589, - 51.969716000000005, - 380, - scn_name, - ) - - drop_line( - 10.004865, - 51.999120000000005, - 9.982829, - 51.985980000000005, - 380, - scn_name, - ) - - drop_line( - 10.174395, - 52.036448, - 9.988215035677026, - 51.954230057487926, - 380, - scn_name, - ) - - drop_line( - 10.195144702845797, - 52.079851837273964, - 10.174395, - 52.036448, - 380, - scn_name, - ) - - drop_trafo(9.988215035677026, 51.954230057487926, 110, 380, scn_name) - - drop_bus(9.988215035677026, 51.954230057487926, 380, scn_name) - drop_bus(9.991477300000001, 51.939711, 380, scn_name) - drop_bus(9.995589, 51.969716000000005, 380, scn_name) - drop_bus(9.982829, 51.985980000000005, 380, scn_name) - drop_bus(10.174395, 52.036448, 380, scn_name) - drop_bus(10.195144702845797, 52.079851837273964, 380, scn_name) - - drop_bus(10.004865, 51.999120000000005, 380, scn_name) - - # Umspannwerk Vieselbach - # delete isolated bus and trafo - drop_trafo(11.121774798935334, 51.00038603925895, 220, 380, scn_name) - drop_bus(11.121774798935334, 51.00038603925895, 380, scn_name) - - # Umspannwerk Waldlaubersheim - # delete isolated bus and trafo - drop_trafo(7.815993836091339, 49.92211102637183, 110, 380, scn_name) - drop_bus(7.815993836091339, 49.92211102637183, 380, scn_name) - def run(): - fix_subnetworks("eGon2035") - fix_subnetworks("eGon100RE") + for scenario in config.settings()["egon-data"]["--scenarios"]: + fix_subnetworks(scenario) diff --git a/src/egon/data/datasets/gas_areas.py b/src/egon/data/datasets/gas_areas.py index d33df886d..c0ccffb90 100755 --- a/src/egon/data/datasets/gas_areas.py +++ b/src/egon/data/datasets/gas_areas.py @@ -1,14 +1,23 @@ -""" -The central module containing code to create CH4 and H2 voronoi polygons +"""The central module containing code to create CH4 and H2 voronoi polygons """ +import datetime +import json + from geoalchemy2.types import Geometry from sqlalchemy import BigInteger, Column, Text from sqlalchemy.ext.declarative import declarative_base -from egon.data import db -from egon.data.datasets import Dataset +from egon.data import db, config +from egon.data.datasets import Dataset, wrapped_partial from egon.data.datasets.generate_voronoi import get_voronoi_geodataframe +from egon.data.metadata import ( + context, + contributors, + license_egon_data_odbl, + meta_metadata, + sources, +) class GasAreaseGon2035(Dataset): @@ -69,6 +78,41 @@ def __init__(self, dependencies): ) +class GasAreasStatusQuo(Dataset): + """Create the gas voronoi table and the gas voronoi areas for status2019 + + *Dependencies* + * :py:class:`EtragoSetup ` + * :py:class:`Vg250 ` + * :py:class:`GasNodesAndPipes ` + + *Resulting tables* + * :py:class:`EgonPfHvGasVoronoi ` + + """ + + #: + name: str = "GasAreasStatusQuo" + #: + version: str = "0.0.2" + + def __init__(self, dependencies): + tasks = (create_gas_voronoi_table,) + + for scn_name in config.settings()["egon-data"]["--scenarios"]: + if "status" in scn_name: + tasks += (wrapped_partial( + voronoi_status, scn_name=scn_name, postfix=f"_{scn_name[-4:]}" + ),) + + super().__init__( + name=self.name, + version=self.version, + dependencies=dependencies, + tasks=tasks, + ) + + Base = declarative_base() @@ -77,8 +121,78 @@ class EgonPfHvGasVoronoi(Base): Class definition of table grid.egon_gas_voronoi """ + + source_list = [ + sources()["openstreetmap"], + sources()["SciGRID_gas"], + sources()["bgr_inspeeds_data_bundle"], + ] + meta = { + "name": "grid.egon_gas_voronoi", + "title": "Gas voronoi areas", + "id": "WILL_BE_SET_AT_PUBLICATION", + "description": "H2 and CH4 voronoi cells", + "language": ["en-EN"], + "publicationDate": datetime.date.today().isoformat(), + "context": context(), + "spatial": { + "location": None, + "extent": "Germany", + "resolution": None, + }, + "sources": source_list, + "licenses": [license_egon_data_odbl()], + "contributors": contributors(["fw"]), + "resources": [ + { + "profile": "tabular-data-resource", + "name": "grid.egon_gas_voronoi", + "path": None, + "format": "PostgreSQL", + "encoding": "UTF-8", + "schema": { + "fields": [ + { + "name": "scn_name", + "description": "Name of the scenario", + "type": "str", + "unit": None, + }, + { + "name": "bus_id", + "description": "Unique identifier", + "type": "integer", + "unit": None, + }, + { + "name": "carrier", + "description": "Carrier of the voronoi cell", + "type": "str", + "unit": None, + }, + { + "name": "geom", + "description": "Voronoi cell geometry", + "type": "Geometry(Polygon, 4326)", + "unit": None, + }, + ], + "primaryKey": ["scn_name", "bus_id"], + "foreignKeys": [], + }, + "dialect": {"delimiter": None, "decimalSeparator": "."}, + } + ], + "metaMetadata": meta_metadata(), + } + # Create json dump + meta_json = "'" + json.dumps(meta, indent=4, ensure_ascii=False) + "'" + __tablename__ = "egon_gas_voronoi" - __table_args__ = {"schema": "grid"} + __table_args__ = { + "schema": "grid", + "comment": meta_json, + } #: Name of the scenario scn_name = Column(Text, primary_key=True, nullable=False) @@ -115,6 +229,14 @@ def voronoi_egon100RE(): create_voronoi("eGon100RE", carrier) +def voronoi_status(scn_name): + """ + Create voronoi polygons for all gas carriers in status_x scenario + """ + for carrier in ["CH4"]: + create_voronoi(scn_name, carrier) + + def create_voronoi(scn_name, carrier): """ Create voronoi polygons for specified carrier in specified scenario. @@ -156,16 +278,24 @@ def create_voronoi(scn_name, carrier): if len(buses) == 0: return - buses["x"] = buses.geometry.x - buses["y"] = buses.geometry.y # generate voronois - gdf = get_voronoi_geodataframe(buses, boundary.geometry.iloc[0]) - # set scn_name + # For some scenarios it is defined that there is only 1 bus (e.g. gas). It + # means that there will be just 1 voronoi covering the entire german + # territory. For other scenarios with more buses, voronois are calculated. + if len(buses) == 1: + gdf = buses.copy() + gdf.at[0, "geom"] = boundary.at[0, "geometry"] + else: + buses["x"] = buses.geometry.x + buses["y"] = buses.geometry.y + gdf = get_voronoi_geodataframe(buses, boundary.geometry.iloc[0]) + gdf.rename_geometry("geom", inplace=True) + gdf.drop(columns=["id"], inplace=True) + + # set scn_name and carrier gdf["scn_name"] = scn_name gdf["carrier"] = carrier - gdf.rename_geometry("geom", inplace=True) - gdf.drop(columns=["id"], inplace=True) # Insert data to db gdf.set_crs(epsg=4326).to_postgis( f"egon_gas_voronoi", diff --git a/src/egon/data/datasets/gas_grid.py b/src/egon/data/datasets/gas_grid.py index 6fc3c18bd..cab2ac4e1 100755 --- a/src/egon/data/datasets/gas_grid.py +++ b/src/egon/data/datasets/gas_grid.py @@ -22,6 +22,7 @@ import json import os +from sqlalchemy.orm import sessionmaker from geoalchemy2.types import Geometry from shapely import geometry import geopandas @@ -30,47 +31,12 @@ from egon.data import config, db from egon.data.config import settings -from egon.data.datasets import Dataset +from egon.data.datasets import Dataset, wrapped_partial from egon.data.datasets.electrical_neighbours import central_buses_egon100 from egon.data.datasets.etrago_helpers import copy_and_modify_buses from egon.data.datasets.scenario_parameters import get_sector_parameters -class GasNodesAndPipes(Dataset): - """ - Insert the CH4 buses and links into the database. - - Insert the CH4 buses and links, which for the case of gas represent - pipelines, into the database for the scenarios eGon2035 and eGon100RE - with the functions :py:func:`insert_gas_data` and :py:func:`insert_gas_data_eGon100RE`. - - *Dependencies* - * :py:class:`DataBundle ` - * :py:class:`ElectricalNeighbours ` - * :py:class:`Osmtgmod ` - * :py:class:`ScenarioParameters ` - * :py:class:`EtragoSetup ` (more specifically the :func:`create_tables ` task) - - *Resulting tables* - * :py:class:`grid.egon_etrago_bus ` is extended - * :py:class:`grid.egon_etrago_link ` is extended - - """ - - #: - name: str = "GasNodesAndPipes" - #: - version: str = "0.0.9" - - def __init__(self, dependencies): - super().__init__( - name=self.name, - version=self.version, - dependencies=dependencies, - tasks=(insert_gas_data, insert_gas_data_eGon100RE), - ) - - def download_SciGRID_gas_data(): """ Download SciGRID_gas IGGIELGN data from Zenodo @@ -352,30 +318,42 @@ def define_gas_buses_abroad(scn_name="eGon2035"): gdf_abroad_buses["bus_id"] = range(new_id, new_id + len(gdf_abroad_buses)) # Add central bus in Russia - gdf_abroad_buses = gdf_abroad_buses.append( - { - "scn_name": scn_name, - "bus_id": (new_id + len(gdf_abroad_buses) + 1), - "x": 41, - "y": 55, - "country": "RU", - "carrier": main_gas_carrier, - }, + gdf_abroad_buses = pd.concat( + [ + gdf_abroad_buses, + pd.DataFrame( + index=[gdf_abroad_buses.index.max() + 1], + data={ + "scn_name": scn_name, + "bus_id": (new_id + len(gdf_abroad_buses) + 1), + "x": 41, + "y": 55, + "country": "RU", + "carrier": main_gas_carrier, + }, + ), + ], ignore_index=True, ) # if in test mode, add bus in center of Germany boundary = settings()["egon-data"]["--dataset-boundary"] if boundary != "Everything": - gdf_abroad_buses = gdf_abroad_buses.append( - { - "scn_name": scn_name, - "bus_id": (new_id + len(gdf_abroad_buses) + 1), - "x": 10.4234469, - "y": 51.0834196, - "country": "DE", - "carrier": main_gas_carrier, - }, + gdf_abroad_buses = pd.concat( + [ + gdf_abroad_buses, + pd.DataFrame( + index=[gdf_abroad_buses.index.max() + 1], + data={ + "scn_name": scn_name, + "bus_id": (new_id + len(gdf_abroad_buses) + 1), + "x": 10.4234469, + "y": 51.0834196, + "country": "DE", + "carrier": main_gas_carrier, + }, + ), + ], ignore_index=True, ) @@ -577,7 +555,6 @@ def define_gas_pipeline_list( boundary = settings()["egon-data"]["--dataset-boundary"] if boundary != "Everything": - gas_pipelines_list = gas_pipelines_list[ gas_pipelines_list["NUTS1_0"].str.contains(map_states[boundary]) | gas_pipelines_list["NUTS1_1"].str.contains(map_states[boundary]) @@ -595,7 +572,6 @@ def define_gas_pipeline_list( length_km = [] for index, row in gas_pipelines_list.iterrows(): - param = ast.literal_eval(row["param"]) diameter.append(param["diameter_mm"]) length_km.append(param["length_km"]) @@ -1025,3 +1001,111 @@ def insert_gas_data_eGon100RE(): index=False, dtype={"geom": Geometry(), "topo": Geometry()}, ) + + +def insert_gas_data_status(scn_name): + """ + Function to deal with the gas network for the status2019 scenario. + For this scenario just one CH4 bus is consider in the center of Germany. + Since OCGTs in the foreign countries are modelled as generators and not + as links between the gas and electricity sectors, CH4 foreign buses are + considered not necessary. + + This function does not require any input. + + Returns + ------- + None. + + """ + + # delete old entries + db.execute_sql( + f""" + DELETE FROM grid.egon_etrago_link + WHERE carrier = 'CH4' AND scn_name = '{scn_name}' + """ + ) + db.execute_sql( + f""" + DELETE FROM grid.egon_etrago_bus + WHERE carrier = 'CH4' AND scn_name = '{scn_name}' + """ + ) + + # Select next id value + new_id = db.next_etrago_id("bus") + + df = pd.DataFrame( + index=[new_id], + data={ + "scn_name": scn_name, + "v_nom": 1, + "carrier": "CH4", + "v_mag_pu_set": 1, + "v_mag_pu_min": 0, + "v_mag_pu_max": np.inf, + "x": 10, + "y": 51, + "country": "DE", + }, + ) + gdf = geopandas.GeoDataFrame( + df, geometry=geopandas.points_from_xy(df.x, df.y, crs=4326) + ).rename_geometry("geom") + + gdf.index.name = "bus_id" + + gdf.reset_index().to_postgis( + "egon_etrago_bus", schema="grid", con=db.engine(), if_exists="append" + ) + + +class GasNodesAndPipes(Dataset): + """ + Insert the CH4 buses and links into the database. + + Insert the CH4 buses and links, which for the case of gas represent + pipelines, into the database for the scenarios status2019, eGon2035 and eGon100RE + with the functions :py:func:`insert_gas_data` and :py:func:`insert_gas_data_eGon100RE`. + + *Dependencies* + * :py:class:`DataBundle ` + * :py:class:`ElectricalNeighbours ` + * :py:class:`Osmtgmod ` + * :py:class:`ScenarioParameters ` + * :py:class:`EtragoSetup ` (more specifically the :func:`create_tables ` task) + + *Resulting tables* + * :py:class:`grid.egon_etrago_bus ` is extended + * :py:class:`grid.egon_etrago_link ` is extended + + """ + + #: + name: str = "GasNodesAndPipes" + #: + version: str = "0.0.11" + + tasks = () + + for scn_name in config.settings()["egon-data"]["--scenarios"]: + if "status" in scn_name: + tasks += (wrapped_partial( + insert_gas_data_status, scn_name=scn_name, postfix=f"_{scn_name[-4:]}" + ),) + # tasks = (insert_gas_data_status,) + + if "eGon2035" in config.settings()["egon-data"]["--scenarios"]: + tasks = tasks + (insert_gas_data,) + + if "eGon100RE" in config.settings()["egon-data"]["--scenarios"]: + tasks = tasks + (insert_gas_data_eGon100RE,) + + def __init__(self, dependencies): + super().__init__( + name=self.name, + version=self.version, + dependencies=dependencies, + tasks=self.tasks, + ) diff --git a/src/egon/data/datasets/gas_neighbours/__init__.py b/src/egon/data/datasets/gas_neighbours/__init__.py index 303b5a3bb..8b4d300f9 100755 --- a/src/egon/data/datasets/gas_neighbours/__init__.py +++ b/src/egon/data/datasets/gas_neighbours/__init__.py @@ -33,13 +33,18 @@ class GasNeighbours(Dataset): def __init__(self, dependencies): super().__init__( name="GasNeighbours", - version="0.0.4", + version="0.0.5", dependencies=dependencies, tasks=( - tyndp_gas_generation, - tyndp_gas_demand, - grid, - insert_ocgt_abroad, - insert_gas_neigbours_eGon100RE, + # tyndp_gas_generation, + # tyndp_gas_demand, + # grid, + # insert_ocgt_abroad, + # insert_gas_neigbours_eGon100RE, + notasks, ), ) + + +def notasks(): + return None diff --git a/src/egon/data/datasets/gas_neighbours/eGon2035.py b/src/egon/data/datasets/gas_neighbours/eGon2035.py index c488657b2..09ae3120c 100755 --- a/src/egon/data/datasets/gas_neighbours/eGon2035.py +++ b/src/egon/data/datasets/gas_neighbours/eGon2035.py @@ -1520,7 +1520,6 @@ def calculate_ocgt_capacities(): return df_ocgt - def insert_ocgt_abroad(): """Insert gas turbine capacities abroad for eGon2035 in the database diff --git a/src/egon/data/datasets/heat_demand/__init__.py b/src/egon/data/datasets/heat_demand/__init__.py index ba2481618..1151353bc 100644 --- a/src/egon/data/datasets/heat_demand/__init__.py +++ b/src/egon/data/datasets/heat_demand/__init__.py @@ -1,4 +1,9 @@ # -*- coding: utf-8 -*- + +# This script is part of eGon-data. + +# license text - to be added. + """ Central module containing all code dealing with the future heat demand import. @@ -14,13 +19,14 @@ # for metadata creation import json import os +import time import zipfile from jinja2 import Template from rasterio.mask import mask # packages for ORM class definition -from sqlalchemy import Column, Float, ForeignKey, Integer, Sequence, String +from sqlalchemy import Column, Float, Integer, Sequence, String from sqlalchemy.ext.declarative import declarative_base import geopandas as gpd @@ -30,12 +36,16 @@ from egon.data import db, subprocess from egon.data.datasets import Dataset from egon.data.datasets.scenario_parameters import ( - EgonScenario, get_sector_parameters, ) +from egon.data.metadata import ( + context, + license_ccby, + meta_metadata, + sources, +) import egon.data.config -# import time class HeatDemandImport(Dataset): @@ -94,7 +104,7 @@ class EgonPetaHeat(Base): ) demand = Column(Float) sector = Column(String) - scenario = Column(String, ForeignKey(EgonScenario.name)) + scenario = Column(String) zensus_population_id = Column(Integer) @@ -370,6 +380,28 @@ def future_heat_demand_germany(scenario_name): if scenario_name == "eGon2015": res_hd_reduction = 1 ser_hd_reduction = 1 + + elif scenario_name == "status2019": + heat_parameters = get_sector_parameters("heat", scenario=scenario_name) + + # Calculate reduction share based on final energy demand and overall demand from Peta for 2015 + res_hd_reduction = ( + heat_parameters["DE_demand_residential_TJ"] / 3600 / 443.788483 + ) + ser_hd_reduction = ( + heat_parameters["DE_demand_service_TJ"] / 3600 / 226.588158 + ) + elif scenario_name == "status2023": + heat_parameters = get_sector_parameters("heat", scenario=scenario_name) # currently data for 2019 is used + # see scenario_paramters/__init__ for this. + + # Calculate reduction share based on final energy demand and overall demand from Peta for 2015 + res_hd_reduction = ( + heat_parameters["DE_demand_residential_TJ"] / 3600 / 443.788483 # TODO status2023 can values stay same? + ) + ser_hd_reduction = ( + heat_parameters["DE_demand_service_TJ"] / 3600 / 226.588158 # TODO status2023 can values stay same? + ) else: heat_parameters = get_sector_parameters("heat", scenario=scenario_name) @@ -578,166 +610,26 @@ def add_metadata(): """ Writes metadata JSON string into table comment. - TODO - ---- - Meta data must be check and adjusted to the egon_data standard: - - Add context - - Meta data for Census Population Table must be added. - - Check how to reference the heat demand adjustment factors """ - - # Prepare variables - license_peta5_0_1 = [ - { - "name": "Creative Commons Attribution 4.0 International", - "title": "CC BY 4.0", - "path": "https://creativecommons.org/licenses/by/4.0/", - "instruction": ( - "You are free: To Share, To Adapt;" - " As long as you: Attribute!" - ), - "attribution": "© Flensburg, Halmstad and Aalborg universities", - } - ] - url_peta = ( - "https://s-eenergies-open-data-euf.hub.arcgis.com/search?" - "categories=seenergies_buildings" - ) - - url_geodatenzentrum = ( - "https://daten.gdz.bkg.bund.de/produkte/vg/" - "vg250_ebenen_0101/2020/vg250_01-01.geo84.shape." - "ebenen.zip" - ) - license_heat = [ - { - # this could be the license of "heat" - "name": "Creative Commons Attribution 4.0 International", - "title": "CC BY 4.0", - "path": "https://creativecommons.org/licenses/by/4.0/", - "instruction": ( - "You are free: To Share, To Adapt;" - " As long as you: Attribute!" - ), - "attribution": "© Europa-Universität Flensburg", # if all agree - } - ] - license_BKG = [ - { - "title": "Datenlizenz Deutschland – Namensnennung – Version 2.0", - "path": "www.govdata.de/dl-de/by-2-0", - "instruction": ( - "Jede Nutzung ist unter den Bedingungen dieser „Datenlizenz " - "Deutschland - Namensnennung - Version 2.0 zulässig.\nDie " - "bereitgestellten Daten und Metadaten dürfen für die " - "kommerzielle und nicht kommerzielle Nutzung insbesondere:" - "(1) vervielfältigt, ausgedruckt, präsentiert, verändert, " - "bearbeitet sowie an Dritte übermittelt werden;\n " - "(2) mit eigenen Daten und Daten Anderer zusammengeführt und " - "zu selbständigen neuen Datensätzen verbunden werden;\n " - "(3) in interne und externe Geschäftsprozesse, Produkte und " - "Anwendungen in öffentlichen und nicht öffentlichen " - "elektronischen Netzwerken eingebunden werden." - ), - "attribution": "© Bundesamt für Kartographie und Geodäsie", - } - ] - # Metadata creation meta = { "name": "egon_peta_heat_metadata", "title": "eGo_n scenario-specific future heat demand data", + "id": "WILL_BE_SET_AT_PUBLICATION", "description": "Future heat demands per hectare grid cell of " "the residential and service sector", "language": ["EN"], + "context": context(), "spatial": { - "location": "", + "location": None, "extent": "Germany", "resolution": "100x100m", }, - "temporal": { - "referenceDate": "scenario-specific", - "timeseries": { - "start": "", - "end": "", - "resolution": "", - "alignment": "", - "aggregationType": "", - }, - }, "sources": [ - { - # for Peta5_0_1 - "title": "Peta5_0_1_HD_res and Peta5 0 1 HD ser", - "description": "Der Datenbestand umfasst sämtliche " - "Verwaltungseinheiten aller hierarchischen " - "Verwaltungsebenen vom Staat bis zu den " - "Gemeinden mit ihren Verwaltungsgrenzen, " - "statistischen Schlüsselzahlen und dem " - "Namen der Verwaltungseinheit sowie der " - "spezifischen Bezeichnung der " - "Verwaltungsebene des jeweiligen " - "Bundeslandes.", - "path": url_peta, - "licenses": license_peta5_0_1, - }, - { - # for the vg250_sta_union used - Please check! - "title": "Dienstleistungszentrum des Bundes für " - "Geoinformation und Geodäsie - Open Data", - "description": "Dieser Datenbestand steht über " - "Geodatendienste gemäß " - "Geodatenzugangsgesetz (GeoZG) " - "(http://www.geodatenzentrum.de/auftrag/pdf" - "/geodatenzugangsgesetz.pdf) für die " - "kommerzielle und nicht kommerzielle " - "Nutzung geldleistungsfrei zum Download " - "und zur Online-Nutzung zur Verfügung. Die " - "Nutzung der Geodaten und Geodatendienste " - "wird durch die Verordnung zur Festlegung " - "der Nutzungsbestimmungen für die " - "Bereitstellung von Geodaten des Bundes " - "(GeoNutzV) (http://www.geodatenzentrum.de" - "/auftrag/pdf/geonutz.pdf) geregelt. " - "Insbesondere hat jeder Nutzer den " - "Quellenvermerk zu allen Geodaten, " - "Metadaten und Geodatendiensten erkennbar " - "und in optischem Zusammenhang zu " - "platzieren. Veränderungen, Bearbeitungen, " - "neue Gestaltungen oder sonstige " - "Abwandlungen sind mit einem " - "Veränderungshinweis im Quellenvermerk zu " - "versehen. Quellenvermerk und " - "Veränderungshinweis sind wie folgt zu " - "gestalten. Bei der Darstellung auf einer " - "Webseite ist der Quellenvermerk mit der " - "URL http://www.bkg.bund.de zu verlinken. " - "© GeoBasis-DE / BKG © GeoBasis-DE / BKG " - " " - "(Daten verändert) Beispiel: " - "© GeoBasis-DE / BKG 2013", - "path": url_geodatenzentrum, - "licenses": "Geodatenzugangsgesetz (GeoZG)", - "copyright": "© GeoBasis-DE / BKG 2016 (Daten verändert)", - }, - { - # for the vg250_sta_union used, too - Please check! - "title": "BKG - Verwaltungsgebiete 1:250.000 (vg250)", - "description": "Der Datenbestand umfasst sämtliche " - "Verwaltungseinheiten aller hierarchischen " - "Verwaltungsebenen vom Staat bis zu den " - "Gemeinden mit ihren Verwaltungsgrenzen, " - "statistischen Schlüsselzahlen und dem " - "Namen der Verwaltungseinheit sowie der " - "spezifischen Bezeichnung der " - "Verwaltungsebene des jeweiligen " - "Bundeslandes.", - "path": "http://www.bkg.bund.de", - "licenses": license_BKG, - }, + sources()["egon-data"], + sources()["peta"], + sources()["vg250"], + sources()["zensus"], ], "resources": [ { @@ -800,24 +692,24 @@ def add_metadata(): "dialect": {"delimiter": "none", "decimalSeparator": "."}, } ], - "licenses": license_heat, + "licenses": [license_ccby("© Europa-Universität Flensburg")], "contributors": [ { - "title": "Eva, Günni, Clara", - "email": "", - "date": "2021-03-04", - "object": "", - "comment": "Processed data", - } - ], - "metaMetadata": { # https://github.com/OpenEnergyPlatform/oemetadata - "metadataVersion": "OEP-1.4.0", - "metadataLicense": { - "name": "CC0-1.0", - "title": "Creative Commons Zero v1.0 Universal", - "path": ("https://creativecommons.org/publicdomain/zero/1.0/"), + "title": "EvaWie", + "email": "http://github.com/EvaWie", + "date": time.strftime("%Y-%m-%d"), + "object": None, + "comment": "Imported data", }, - }, + { + "title": "Clara Büttner", + "email": "http://github.com/ClaraBuettner", + "date": time.strftime("%Y-%m-%d"), + "object": None, + "comment": "Updated metadata", + }, + ], + "metaMetadata": meta_metadata(), } meta_json = "'" + json.dumps(meta) + "'" diff --git a/src/egon/data/datasets/heat_demand_timeseries/__init__.py b/src/egon/data/datasets/heat_demand_timeseries/__init__.py index 3a8414a1b..c0f6b4e28 100644 --- a/src/egon/data/datasets/heat_demand_timeseries/__init__.py +++ b/src/egon/data/datasets/heat_demand_timeseries/__init__.py @@ -1,6 +1,8 @@ -from datetime import datetime +from datetime import date, datetime from pathlib import Path +import json import os +import time from sqlalchemy import ARRAY, Column, Float, Integer, String, Text from sqlalchemy.ext.declarative import declarative_base @@ -8,7 +10,7 @@ import numpy as np import pandas as pd -from egon.data import db +from egon.data import config, db import egon.data.datasets.era5 as era try: @@ -27,7 +29,12 @@ from egon.data.datasets.heat_demand_timeseries.service_sector import ( CTS_demand_scale, ) -import egon +from egon.data.metadata import ( + context, + license_egon_data_odbl, + meta_metadata, + sources, +) Base = declarative_base() @@ -87,7 +94,11 @@ def create_timeseries_for_building(building_id, scenario): SELECT building_demand * UNNEST(idp) as demand FROM ( - SELECT demand.demand / building.count * daily_demand.daily_demand_share as building_demand, daily_demand.day_of_year + SELECT + demand.demand + / building.count + * daily_demand.daily_demand_share as building_demand, + daily_demand.day_of_year FROM (SELECT demand FROM @@ -111,8 +122,12 @@ def create_timeseries_for_building(building_id, scenario): WHERE climate_zone = ( SELECT climate_zone FROM boundaries.egon_map_zensus_climate_zones WHERE zensus_population_id = - (SELECT zensus_population_id FROM demand.egon_heat_timeseries_selected_profiles - WHERE building_id = {building_id}))) as daily_demand) as daily_demand + ( + SELECT zensus_population_id + FROM demand.egon_heat_timeseries_selected_profiles + WHERE building_id = {building_id} + ) + )) as daily_demand) as daily_demand JOIN (SELECT b.idp, ordinality as day FROM demand.egon_heat_timeseries_selected_profiles a, @@ -126,20 +141,23 @@ def create_timeseries_for_building(building_id, scenario): def create_district_heating_profile(scenario, area_id): - """Create heat demand profile for district heating grid including demands of - households and service sector. + """Create a heat demand profile for a district heating grid. + + The created heat demand profile includes the demands of households + and the service sector. Parameters ---------- scenario : str - Name of the selected scenario. + The name of the selected scenario. area_id : int - Index of the selected district heating grid + The index of the selected district heating grid. Returns ------- - df : pandas,DataFrame - Hourly heat demand timeseries in MW for the selected district heating grid + pd.DataFrame + An hourly heat demand timeseries in MW for the selected district + heating grid. """ @@ -178,7 +196,8 @@ def create_district_heating_profile(scenario, area_id): JOIN demand.egon_daily_heat_demand_per_climate_zone c ON c.climate_zone = b.climate_zone - JOIN (SELECT e.idp, ordinality as day, zensus_population_id, building_id + JOIN ( + SELECT e.idp, ordinality as day, zensus_population_id, building_id FROM demand.egon_heat_timeseries_selected_profiles d, UNNEST (d.selected_idp_profiles) WITH ORDINALITY as selected_idp JOIN demand.egon_heat_idp_pool e @@ -203,7 +222,7 @@ def create_district_heating_profile(scenario, area_id): WHERE scenario = '{scenario}' AND area_id = {area_id} )) - GROUP BY zensus_population_id) building + GROUP BY zensus_population_id) building ON building.zensus_population_id = b.zensus_population_id, UNNEST(demand_profile.idp) WITH ORDINALITY as hourly_demand @@ -216,9 +235,9 @@ def create_district_heating_profile(scenario, area_id): ) print( - f"Time to create time series for district heating grid {scenario} {area_id}:" + f"Time to create time series for district heating grid {scenario}" + f" {area_id}:\n{datetime.now() - start_time}" ) - print(datetime.now() - start_time) return df @@ -258,7 +277,12 @@ def create_district_heating_profile_python_like(scenario="eGon2035"): annual_demand = db.select_dataframe( f""" - SELECT a.zensus_population_id, demand/c.count as per_building , area_id, demand as demand_total FROM + SELECT + a.zensus_population_id, + demand / c.count as per_building, + area_id, + demand as demand_total + FROM demand.egon_peta_heat a INNER JOIN ( SELECT * FROM demand.egon_map_zensus_district_heating_areas @@ -299,104 +323,113 @@ def create_district_heating_profile_python_like(scenario="eGon2035"): aggregation_level="district" ) - # TODO: use session_scope! - from sqlalchemy.orm import sessionmaker - - session = sessionmaker(bind=db.engine())() - print(datetime.now() - start_time) start_time = datetime.now() for area in district_heating_grids.area_id.unique(): - selected_profiles = db.select_dataframe( - f""" - SELECT a.zensus_population_id, building_id, c.climate_zone, - selected_idp, ordinality as day, b.area_id - FROM demand.egon_heat_timeseries_selected_profiles a - INNER JOIN boundaries.egon_map_zensus_climate_zones c - ON a.zensus_population_id = c.zensus_population_id - INNER JOIN ( - SELECT * FROM demand.egon_map_zensus_district_heating_areas - WHERE scenario = '{scenario}' - AND area_id = '{area}' - ) b ON a.zensus_population_id = b.zensus_population_id , - - UNNEST (selected_idp_profiles) WITH ORDINALITY as selected_idp - """ - ) + with db.session_scope() as session: + + sql = f""" + SELECT a.zensus_population_id, building_id, c.climate_zone, + selected_idp, ordinality as day, b.area_id + FROM demand.egon_heat_timeseries_selected_profiles a + INNER JOIN boundaries.egon_map_zensus_climate_zones c + ON a.zensus_population_id = c.zensus_population_id + INNER JOIN ( + SELECT * FROM demand.egon_map_zensus_district_heating_areas + WHERE scenario = '{scenario}' + AND area_id = '{area}' + ) b ON a.zensus_population_id = b.zensus_population_id, + UNNEST (selected_idp_profiles) WITH ORDINALITY as selected_idp + """ + + selected_profiles = db.select_dataframe(sql) + + if not selected_profiles.empty: + df = pd.merge( + selected_profiles, + daily_demand_shares, + on=["day", "climate_zone"], + ) - if not selected_profiles.empty: + slice_df = pd.merge( + df[df.area_id == area], + idp_df, + left_on="selected_idp", + right_on="index", + ) - df = pd.merge( - selected_profiles, - daily_demand_shares, - on=["day", "climate_zone"], - ) + for hour in range(24): + slice_df[hour] = ( + slice_df.idp.str[hour] + .mul(slice_df.daily_demand_share) + .mul( + annual_demand.loc[ + slice_df.zensus_population_id.values, + "per_building", + ].values + ) + ) - slice_df = pd.merge( - df[df.area_id == area], - idp_df, - left_on="selected_idp", - right_on="index", - ) + diff = ( + slice_df[range(24)].sum().sum() + - annual_demand[ + annual_demand.area_id == area + ].demand_total.sum() + ) / ( + annual_demand[annual_demand.area_id == area].demand_total.sum() + ) - for hour in range(24): - slice_df[hour] = ( - slice_df.idp.str[hour] - .mul(slice_df.daily_demand_share) - .mul( - annual_demand.loc[ - slice_df.zensus_population_id.values, - "per_building", - ].values + assert ( + abs(diff) < 0.04 + ), f"""Deviation of residential heat demand time + series for district heating grid {str(area)} is {diff}""" + + hh = np.concatenate( + slice_df.drop( + [ + "zensus_population_id", + "building_id", + "climate_zone", + "selected_idp", + "area_id", + "daily_demand_share", + "idp", + ], + axis="columns", ) + .groupby("day") + .sum()[range(24)] + .values + ).ravel() + + cts = CTS_demand_dist[ + (CTS_demand_dist.scenario == scenario) + & (CTS_demand_dist.index == area) + ].drop("scenario", axis="columns") + + if (not selected_profiles.empty) and not cts.empty: + entry = EgonTimeseriesDistrictHeating( + area_id=int(area), + scenario=scenario, + dist_aggregated_mw=(hh + cts.values[0]).tolist(), + ) + elif (not selected_profiles.empty) and cts.empty: + entry = EgonTimeseriesDistrictHeating( + area_id=int(area), + scenario=scenario, + dist_aggregated_mw=(hh).tolist(), + ) + elif not cts.empty: + entry = EgonTimeseriesDistrictHeating( + area_id=int(area), + scenario=scenario, + dist_aggregated_mw=(cts.values[0]).tolist(), ) - diff = ( - slice_df.groupby("day").sum()[range(24)].sum().sum() - - annual_demand[ - annual_demand.area_id == area - ].demand_total.sum() - ) / ( - annual_demand[annual_demand.area_id == area].demand_total.sum() - ) - - assert ( - abs(diff) < 0.03 - ), f"""Deviation of residential heat demand time - series for district heating grid {str(area)} is {diff}""" - - hh = np.concatenate( - slice_df.groupby("day").sum()[range(24)].values - ).ravel() - - cts = CTS_demand_dist[ - (CTS_demand_dist.scenario == scenario) - & (CTS_demand_dist.index == area) - ].drop("scenario", axis="columns") - - if (not selected_profiles.empty) and not cts.empty: - entry = EgonTimeseriesDistrictHeating( - area_id=int(area), - scenario=scenario, - dist_aggregated_mw=(hh + cts.values[0]).tolist(), - ) - elif (not selected_profiles.empty) and cts.empty: - entry = EgonTimeseriesDistrictHeating( - area_id=int(area), - scenario=scenario, - dist_aggregated_mw=(hh).tolist(), - ) - elif not cts.empty: - entry = EgonTimeseriesDistrictHeating( - area_id=int(area), - scenario=scenario, - dist_aggregated_mw=(cts.values[0]).tolist(), - ) - - session.add(entry) - session.commit() + session.add(entry) + session.commit() print( f"Time to create time series for district heating scenario {scenario}" @@ -438,7 +471,9 @@ def create_individual_heat_per_mv_grid(scenario="eGon2035", mv_grid_id=1564): JOIN demand.egon_daily_heat_demand_per_climate_zone c ON c.climate_zone = b.climate_zone - JOIN (SELECT e.idp, ordinality as day, zensus_population_id, building_id + JOIN ( + SELECT + e.idp, ordinality as day, zensus_population_id, building_id FROM demand.egon_heat_timeseries_selected_profiles d, UNNEST (d.selected_idp_profiles) WITH ORDINALITY as selected_idp JOIN demand.egon_heat_idp_pool e @@ -461,7 +496,7 @@ def create_individual_heat_per_mv_grid(scenario="eGon2035", mv_grid_id=1564): boundaries.egon_map_zensus_grid_districts WHERE bus_id = {mv_grid_id} )) - GROUP BY zensus_population_id) building + GROUP BY zensus_population_id) building ON building.zensus_population_id = b.zensus_population_id, UNNEST(demand_profile.idp) WITH ORDINALITY as hourly_demand @@ -480,7 +515,6 @@ def create_individual_heat_per_mv_grid(scenario="eGon2035", mv_grid_id=1564): def calulate_peak_load(df, scenario): - # peat load in W_th data = ( df.groupby("building_id") @@ -505,7 +539,6 @@ def calulate_peak_load(df, scenario): def create_individual_heating_peak_loads(scenario="eGon2035"): - engine = db.engine() EgonIndividualHeatingPeakLoads.__table__.drop(bind=engine, checkfirst=True) @@ -547,7 +580,8 @@ def create_individual_heating_peak_loads(scenario="eGon2035"): WHERE a.scenario = '{scenario}' AND a.sector = 'residential' AND a.zensus_population_id NOT IN ( - SELECT zensus_population_id FROM demand.egon_map_zensus_district_heating_areas + SELECT zensus_population_id + FROM demand.egon_map_zensus_district_heating_areas WHERE scenario = '{scenario}' ) @@ -564,7 +598,6 @@ def create_individual_heating_peak_loads(scenario="eGon2035"): start_time = datetime.now() for grid in annual_demand.bus_id.unique(): - selected_profiles = db.select_dataframe( f""" SELECT a.zensus_population_id, building_id, c.climate_zone, @@ -577,7 +610,8 @@ def create_individual_heating_peak_loads(scenario="eGon2035"): UNNEST (selected_idp_profiles) WITH ORDINALITY as selected_idp WHERE a.zensus_population_id NOT IN ( - SELECT zensus_population_id FROM demand.egon_map_zensus_district_heating_areas + SELECT zensus_population_id + FROM demand.egon_map_zensus_district_heating_areas WHERE scenario = '{scenario}' ) AND a.zensus_population_id IN ( @@ -615,7 +649,6 @@ def create_individual_heating_peak_loads(scenario="eGon2035"): def create_individual_heating_profile_python_like(scenario="eGon2035"): - start_time = datetime.now() idp_df = db.select_dataframe( @@ -627,7 +660,11 @@ def create_individual_heating_profile_python_like(scenario="eGon2035"): annual_demand = db.select_dataframe( f""" - SELECT a.zensus_population_id, demand/c.count as per_building, demand as demand_total, bus_id + SELECT + a.zensus_population_id, + demand / c.count as per_building, + demand as demand_total, + bus_id FROM demand.egon_peta_heat a @@ -649,7 +686,8 @@ def create_individual_heating_profile_python_like(scenario="eGon2035"): WHERE a.scenario = '{scenario}' AND a.sector = 'residential' AND a.zensus_population_id NOT IN ( - SELECT zensus_population_id FROM demand.egon_map_zensus_district_heating_areas + SELECT zensus_population_id + FROM demand.egon_map_zensus_district_heating_areas WHERE scenario = '{scenario}' ) @@ -658,7 +696,7 @@ def create_individual_heating_profile_python_like(scenario="eGon2035"): ) daily_demand_shares = db.select_dataframe( - f""" + """ SELECT climate_zone, day_of_year as day, daily_demand_share FROM demand.egon_daily_heat_demand_per_climate_zone """ @@ -680,7 +718,6 @@ def create_individual_heating_profile_python_like(scenario="eGon2035"): start_time = datetime.now() for grid in annual_demand.bus_id.unique(): - selected_profiles = db.select_dataframe( f""" SELECT a.zensus_population_id, building_id, c.climate_zone, @@ -738,10 +775,10 @@ def create_individual_heating_profile_python_like(scenario="eGon2035"): - annual_demand[annual_demand.bus_id == grid].demand_total.sum() ) / (annual_demand[annual_demand.bus_id == grid].demand_total.sum()) - assert ( - abs(diff) < 0.03 - ), f"""Deviation of residential heat demand time - series for mv grid {str(grid)} is {diff}""" + assert abs(diff) < 0.03, ( + "Deviation of residential heat demand time series for mv" + f" grid {grid} is {diff}" + ) if not (slice_df[hour].empty or cts.empty): entry = EgonEtragoTimeseriesIndividualHeating( @@ -773,7 +810,6 @@ def create_individual_heating_profile_python_like(scenario="eGon2035"): def district_heating(method="python"): - engine = db.engine() EgonTimeseriesDistrictHeating.__table__.drop(bind=engine, checkfirst=True) EgonTimeseriesDistrictHeating.__table__.create( @@ -781,11 +817,12 @@ def district_heating(method="python"): ) if method == "python": - create_district_heating_profile_python_like("eGon2035") - create_district_heating_profile_python_like("eGon100RE") + for scenario in config.settings()["egon-data"][ + "--scenarios" + ]: + create_district_heating_profile_python_like(scenario) else: - CTS_demand_dist, CTS_demand_grid, CTS_demand_zensus = CTS_demand_scale( aggregation_level="district" ) @@ -861,7 +898,6 @@ def individual_heating_per_mv_grid_100(method="python"): def individual_heating_per_mv_grid(method="python"): - if method == "python": engine = db.engine() EgonEtragoTimeseriesIndividualHeating.__table__.drop( @@ -875,7 +911,6 @@ def individual_heating_per_mv_grid(method="python"): create_individual_heating_profile_python_like("eGon100RE") else: - engine = db.engine() EgonEtragoTimeseriesIndividualHeating.__table__.drop( bind=engine, checkfirst=True @@ -897,7 +932,6 @@ def individual_heating_per_mv_grid(method="python"): ) for index, row in ids.iterrows(): - for scenario in ["eGon2035", "eGon100RE"]: series = create_individual_heat_per_mv_grid( scenario, row.bus_id @@ -938,7 +972,6 @@ def individual_heating_per_mv_grid(method="python"): def store_national_profiles(): - scenario = "eGon100RE" df = db.select_dataframe( @@ -969,7 +1002,8 @@ def store_national_profiles(): JOIN demand.egon_daily_heat_demand_per_climate_zone c ON c.climate_zone = b.climate_zone - JOIN (SELECT e.idp, ordinality as day, zensus_population_id, building_id + JOIN ( + SELECT e.idp, ordinality as day, zensus_population_id, building_id FROM demand.egon_heat_timeseries_selected_profiles d, UNNEST (d.selected_idp_profiles) WITH ORDINALITY as selected_idp JOIN demand.egon_heat_idp_pool e @@ -984,7 +1018,7 @@ def store_national_profiles(): SELECT zensus_population_id FROM demand.egon_heat_timeseries_selected_profiles ) - GROUP BY zensus_population_id) building + GROUP BY zensus_population_id) building ON building.zensus_population_id = b.zensus_population_id, UNNEST(demand_profile.idp) WITH ORDINALITY as hourly_demand @@ -1062,6 +1096,85 @@ def export_etrago_cts_heat_profiles(): ) +def metadata(): + fields = [ + { + "description": "Index of corresponding district heating area", + "name": "area_id", + "type": "integer", + "unit": "none", + }, + { + "description": "Name of scenario", + "name": "scenario", + "type": "str", + "unit": "none", + }, + { + "description": "Heat demand time series", + "name": "dist_aggregated_mw", + "type": "array of floats", + "unit": "MW", + }, + ] + + meta_district = { + "name": "demand.egon_timeseries_district_heating", + "title": "eGon heat demand time series for district heating grids", + "id": "WILL_BE_SET_AT_PUBLICATION", + "description": "Heat demand time series for district heating grids", + "language": ["EN"], + "publicationDate": date.today().isoformat(), + "context": context(), + "spatial": { + "location": None, + "extent": "Germany", + "resolution": None, + }, + "sources": [ + sources()["era5"], + sources()["vg250"], + sources()["egon-data"], + sources()["egon-data_bundle"], + sources()["peta"], + ], + "licenses": [license_egon_data_odbl()], + "contributors": [ + { + "title": "Clara Büttner", + "email": "http://github.com/ClaraBuettner", + "date": time.strftime("%Y-%m-%d"), + "object": None, + "comment": "Imported data", + }, + ], + "resources": [ + { + "profile": "tabular-data-resource", + "name": "demand.egon_timeseries_district_heating", + "path": None, + "format": "PostgreSQL", + "encoding": "UTF-8", + "schema": { + "fields": fields, + "primaryKey": ["index"], + "foreignKeys": [], + }, + "dialect": {"delimiter": None, "decimalSeparator": "."}, + } + ], + "metaMetadata": meta_metadata(), + } + + # Add metadata as a comment to the table + db.submit_comment( + "'" + json.dumps(meta_district) + "'", + EgonTimeseriesDistrictHeating.__table__.schema, + EgonTimeseriesDistrictHeating.__table__.name, + ) + + + class HeatTimeSeries(Dataset): """ Chooses heat demand profiles for each residential and CTS building @@ -1103,7 +1216,7 @@ class HeatTimeSeries(Dataset): #: name: str = "HeatTimeSeries" #: - version: str = "0.0.7" + version: str = "0.0.8" def __init__(self, dependencies): super().__init__( @@ -1119,6 +1232,7 @@ def __init__(self, dependencies): }, select, district_heating, + metadata, # store_national_profiles, ), ) diff --git a/src/egon/data/datasets/heat_demand_timeseries/daily.py b/src/egon/data/datasets/heat_demand_timeseries/daily.py index 9217bd9f8..d98211cd0 100644 --- a/src/egon/data/datasets/heat_demand_timeseries/daily.py +++ b/src/egon/data/datasets/heat_demand_timeseries/daily.py @@ -9,6 +9,7 @@ from egon.data import db import egon.data.datasets.era5 as era +from egon.data.datasets.scenario_parameters import get_sector_parameters from math import ceil @@ -155,7 +156,7 @@ def map_climate_zones_to_zensus(): def daily_demand_shares_per_climate_zone(): - """Calculates shares of heat demand per day for each cliamte zone + """Calculates shares of heat demand per day for each climate zone Returns ------- @@ -171,14 +172,17 @@ def daily_demand_shares_per_climate_zone(): bind=engine, checkfirst=True ) + # Get temperature profiles of all TRY Climate Zones 2011 + temp_profile = temperature_profile_extract() + # Calulate daily demand shares - h = h_value() + h = h_value(temp_profile) # Normalize data to sum()=1 daily_demand_shares = h.resample("d").sum() / h.sum() # Extract temperature class for each day and climate zone - temperature_classes = temp_interval().resample("D").max() + temperature_classes = temp_interval(temp_profile).resample("D").max() # Initilize dataframe df = pd.DataFrame( @@ -192,16 +196,21 @@ def daily_demand_shares_per_climate_zone(): # Insert data into dataframe for index, row in daily_demand_shares.transpose().iterrows(): - - df = df.append( - pd.DataFrame( - data={ - "climate_zone": index, - "day_of_year": row.index.day_of_year, - "daily_demand_share": row.values, - "temperature_class": temperature_classes[index][row.index], - } - ) + df = pd.concat( + [ + df, + pd.DataFrame( + data={ + "climate_zone": index, + "day_of_year": row.index.day_of_year, + "daily_demand_share": row.values, + "temperature_class": temperature_classes[index][ + row.index + ], + } + ), + ], + ignore_index=True, ) # Insert dataframe to SQL table @@ -312,18 +321,22 @@ def temperature_profile_extract(): return temperature_profile -def temp_interval(): +def temp_interval(temp_profile): """ Description: Create Dataframe with temperature data for TRY Climate Zones + + temp_profile: pandas.DataFrame + temperature profiles of all TRY Climate Zones 2011 Returns ------- temperature_interval : pandas.DataFrame Hourly temperature intrerval of all 15 TRY Climate station#s temperature profile """ - index = pd.date_range(datetime(2011, 1, 1, 0), periods=8760, freq="H") + weather_year = get_sector_parameters("global", "status2023")["weather_year"] + # TODO" status2023 this is currenlty fixed to one scenario possible as only one weather year is possible + index = pd.date_range(datetime(weather_year, 1, 1, 0), periods=8760, freq="H") temperature_interval = pd.DataFrame() - temp_profile = temperature_profile_extract() for x in range(len(temp_profile.columns)): name_station = temp_profile.columns[x] @@ -337,10 +350,12 @@ def temp_interval(): return temperature_interval -def h_value(): +def h_value(temp_profile): """ Description: Assignment of daily demand scaling factor to each day of all TRY Climate Zones + temp_profile: pandas.DataFrame + temperature profiles of all TRY Climate Zones 2011 Returns ------- h : pandas.DataFrame @@ -348,7 +363,10 @@ def h_value(): Extracted from demandlib. """ - index = pd.date_range(datetime(2011, 1, 1, 0), periods=8760, freq="H") + + weather_year = get_sector_parameters("global", "status2023")["weather_year"] + # TODO status2023: this is fixed to 2023 as only one weather year is currently possible + index = pd.date_range(datetime(weather_year, 1, 1, 0), periods=8760, freq="H") a = 3.0469695 @@ -358,7 +376,6 @@ def h_value(): d = 0.1163157 - temp_profile = temperature_profile_extract() temperature_profile_res = ( temp_profile.resample("D") .mean() diff --git a/src/egon/data/datasets/heat_demand_timeseries/idp_pool.py b/src/egon/data/datasets/heat_demand_timeseries/idp_pool.py index d3294ad15..9202fffe0 100644 --- a/src/egon/data/datasets/heat_demand_timeseries/idp_pool.py +++ b/src/egon/data/datasets/heat_demand_timeseries/idp_pool.py @@ -324,20 +324,23 @@ def create(): idp_df = pd.DataFrame(columns=["idp", "house", "temperature_class"]) for s in stock: for m in class_list: - if s == "SFH": i = class_list.index(m) if s == "MFH": i = class_list.index(m) + 9 current_pool = idp_list[i] - idp_df = idp_df.append( - pd.DataFrame( - data={ - "idp": current_pool.transpose().values.tolist(), - "house": s, - "temperature_class": m, - } - ) + idp_df = pd.concat( + [ + idp_df, + pd.DataFrame( + data={ + "idp": current_pool.transpose().values.tolist(), + "house": s, + "temperature_class": m, + } + ), + ], + ignore_index=True, ) idp_df = idp_df.reset_index(drop=True) @@ -362,7 +365,7 @@ def create(): return idp_df -def annual_demand_generator(): +def annual_demand_generator(scenario): """ Description: Create dataframe with annual demand and household count for each zensus cell @@ -374,7 +377,6 @@ def annual_demand_generator(): """ - scenario = "eGon2035" demand_zone = db.select_dataframe( f""" SELECT a.demand, a.zensus_population_id, a.scenario, c.climate_zone @@ -463,7 +465,9 @@ def select(): ) # Calculate annual heat demand per census cell - annual_demand = annual_demand_generator() + annual_demand = annual_demand_generator( + scenario=egon.data.config.settings()["egon-data"]["--scenarios"][0] + ) # Count number of SFH and MFH per climate zone houses_per_climate_zone = ( @@ -476,7 +480,6 @@ def select(): ) for station in houses_per_climate_zone.index: - result_SFH = pd.DataFrame(columns=range(1, 366)) result_MFH = pd.DataFrame(columns=range(1, 366)) diff --git a/src/egon/data/datasets/heat_demand_timeseries/service_sector.py b/src/egon/data/datasets/heat_demand_timeseries/service_sector.py index 2d93d6f50..9cc04b2e9 100644 --- a/src/egon/data/datasets/heat_demand_timeseries/service_sector.py +++ b/src/egon/data/datasets/heat_demand_timeseries/service_sector.py @@ -5,7 +5,7 @@ import pandas as pd -from egon.data import db +from egon.data import config, db try: @@ -74,7 +74,7 @@ def cts_demand_per_aggregation_level(aggregation_level, scenario): df_CTS_gas_2011 = df_CTS_gas_2011.asfreq("H") else: df_CTS_gas_2011 = temporal.disagg_temporal_gas_CTS( - use_nuts3code=True, year=2011 + use_nuts3code=True, year=2017 ) df_CTS_gas_2011.to_csv("CTS_heat_demand_profile_nuts3.csv") @@ -215,7 +215,7 @@ def CTS_demand_scale(aggregation_level): Profiles scaled up to annual demand """ - scenarios = ["eGon2035", "eGon100RE"] + scenarios = config.settings()["egon-data"]["--scenarios"] CTS_district = pd.DataFrame() CTS_grid = pd.DataFrame() @@ -242,7 +242,6 @@ def CTS_demand_scale(aggregation_level): ) if aggregation_level == "district": - district_heating = db.select_dataframe( f""" SELECT area_id, zensus_population_id @@ -283,7 +282,9 @@ def CTS_demand_scale(aggregation_level): CTS_per_district.insert(0, "scenario", scenario) - CTS_district = CTS_district.append(CTS_per_district) + CTS_district = pd.concat( + [CTS_district, CTS_per_district], ignore_index=True + ) CTS_district = CTS_district.sort_index() mv_grid_ind = db.select_dataframe( @@ -332,7 +333,7 @@ def CTS_demand_scale(aggregation_level): CTS_per_grid.insert(0, "scenario", scenario) - CTS_grid = CTS_grid.append(CTS_per_grid) + CTS_grid = pd.concat([CTS_grid, CTS_per_grid]) CTS_grid = CTS_grid.sort_index() CTS_per_zensus = 0 @@ -358,7 +359,7 @@ def CTS_demand_scale(aggregation_level): CTS_per_zensus.reset_index(inplace=True) - CTS_zensus = CTS_zensus.append(CTS_per_grid) + CTS_zensus = pd.concat([CTS_zensus, CTS_per_grid]) CTS_zensus = CTS_zensus.set_index("bus_id") CTS_zensus = CTS_zensus.sort_index() diff --git a/src/egon/data/datasets/heat_etrago/__init__.py b/src/egon/data/datasets/heat_etrago/__init__.py index 2ff427f5c..55d7abc13 100644 --- a/src/egon/data/datasets/heat_etrago/__init__.py +++ b/src/egon/data/datasets/heat_etrago/__init__.py @@ -183,10 +183,10 @@ def insert_store(scenario, carrier): "bus0": dh_bus.bus_id, "bus1": water_tank_bus.bus_id, "carrier": carrier + "_store_charger", - "efficiency": get_sector_parameters("heat", "eGon2035")[ + "efficiency": get_sector_parameters("heat", scenario)[ "efficiency" ]["water_tank_charger"], - "marginal_cost": get_sector_parameters("heat", "eGon2035")[ + "marginal_cost": get_sector_parameters("heat", scenario)[ "marginal_cost" ]["water_tank_charger"], "p_nom_extendable": True, @@ -211,10 +211,10 @@ def insert_store(scenario, carrier): "bus0": water_tank_bus.bus_id, "bus1": dh_bus.bus_id, "carrier": carrier + "_store_discharger", - "efficiency": get_sector_parameters("heat", "eGon2035")[ + "efficiency": get_sector_parameters("heat", scenario)[ "efficiency" ]["water_tank_discharger"], - "marginal_cost": get_sector_parameters("heat", "eGon2035")[ + "marginal_cost": get_sector_parameters("heat", scenario)[ "marginal_cost" ]["water_tank_discharger"], "p_nom_extendable": True, @@ -238,10 +238,10 @@ def insert_store(scenario, carrier): "scn_name": scenario, "bus": water_tank_bus.bus_id, "carrier": carrier + "_store", - "capital_cost": get_sector_parameters("heat", "eGon2035")[ + "capital_cost": get_sector_parameters("heat", scenario)[ "capital_cost" ][f"{carrier.split('_')[0]}_water_tank"], - "lifetime": get_sector_parameters("heat", "eGon2035")["lifetime"][ + "lifetime": get_sector_parameters("heat", scenario)["lifetime"][ f"{carrier.split('_')[0]}_water_tank" ], "e_nom_extendable": True, @@ -262,17 +262,19 @@ def insert_store(scenario, carrier): def store(): - insert_store("eGon2035", "central_heat") - insert_store("eGon2035", "rural_heat") + for scenario in config.settings()["egon-data"]["--scenarios"]: + if "status" not in scenario: + insert_store(scenario, "central_heat") + insert_store(scenario, "rural_heat") -def insert_central_direct_heat(scenario="eGon2035"): +def insert_central_direct_heat(scenario): """Insert renewable heating technologies (solar and geo thermal) Parameters ---------- - scenario : str, optional - Name of the scenario The default is 'eGon2035'. + scenario : str + Name of the scenario Returns ------- @@ -366,13 +368,15 @@ def insert_central_direct_heat(scenario="eGon2035"): # Map solar thermal collectors to weather cells join = gpd.sjoin(weather_cells, solar_thermal)[["index_right"]] + weather_year = get_sector_parameters("global", scenario)["weather_year"] + feedin = db.select_dataframe( f""" SELECT w_id, feedin FROM {sources['feedin_timeseries']['schema']}. {sources['feedin_timeseries']['table']} WHERE carrier = 'solar_thermal' - AND weather_year = 2011 + AND weather_year = {weather_year} """, index_col="w_id", ) @@ -405,13 +409,13 @@ def insert_central_direct_heat(scenario="eGon2035"): ) -def insert_central_gas_boilers(scenario="eGon2035"): +def insert_central_gas_boilers(scenario): """Inserts gas boilers for district heating to eTraGo-table Parameters ---------- - scenario : str, optional - Name of the scenario. The default is 'eGon2035'. + scenario : str + Name of the scenario. Returns ------- @@ -456,11 +460,11 @@ def insert_central_gas_boilers(scenario="eGon2035"): central_boilers = link_geom_from_buses(central_boilers, scenario) # Add efficiency and marginal costs of gas boilers - central_boilers["efficiency"] = get_sector_parameters("heat", "eGon2035")[ + central_boilers["efficiency"] = get_sector_parameters("heat", scenario)[ "efficiency" ]["central_gas_boiler"] central_boilers["marginal_cost"] = get_sector_parameters( - "heat", "eGon2035" + "heat", scenario )["marginal_cost"]["central_gas_boiler"] # Transform thermal capacity to CH4 installed capacity @@ -486,13 +490,13 @@ def insert_central_gas_boilers(scenario="eGon2035"): ) -def insert_rural_gas_boilers(scenario="eGon2035"): +def insert_rural_gas_boilers(scenario): """Inserts gas boilers for individual heating to eTraGo-table Parameters ---------- - scenario : str, optional - Name of the scenario. The default is 'eGon2035'. + scenario : str + Name of the scenario. Returns ------- @@ -549,7 +553,7 @@ def insert_rural_gas_boilers(scenario="eGon2035"): rural_boilers = link_geom_from_buses(rural_boilers, scenario) # Add efficiency of gas boilers - rural_boilers["efficiency"] = get_sector_parameters("heat", "eGon2035")[ + rural_boilers["efficiency"] = get_sector_parameters("heat", scenario)[ "efficiency" ]["rural_gas_boiler"] @@ -587,11 +591,9 @@ def buses(): None. """ - - insert_buses("central_heat", scenario="eGon2035") - insert_buses("rural_heat", scenario="eGon2035") - insert_buses("central_heat", scenario="eGon100RE") - insert_buses("rural_heat", scenario="eGon100RE") + for scenario in config.settings()["egon-data"]["--scenarios"]: + insert_buses("central_heat", scenario) + insert_buses("rural_heat", scenario) def supply(): @@ -606,12 +608,13 @@ def supply(): """ - insert_central_direct_heat(scenario="eGon2035") - insert_central_power_to_heat(scenario="eGon2035") - insert_individual_power_to_heat(scenario="eGon2035") - - # insert_rural_gas_boilers(scenario="eGon2035") - insert_central_gas_boilers(scenario="eGon2035") + for scenario in config.settings()["egon-data"]["--scenarios"]: + # There is no direct heat in status quo scenario + if "status" not in scenario: + insert_central_direct_heat(scenario) + insert_central_power_to_heat(scenario) + insert_individual_power_to_heat(scenario) + insert_central_gas_boilers(scenario) class HeatEtrago(Dataset): diff --git a/src/egon/data/datasets/heat_etrago/hts_etrago.py b/src/egon/data/datasets/heat_etrago/hts_etrago.py index 112919457..eb9daba98 100644 --- a/src/egon/data/datasets/heat_etrago/hts_etrago.py +++ b/src/egon/data/datasets/heat_etrago/hts_etrago.py @@ -10,16 +10,16 @@ import numpy as np -def hts_to_etrago(): - +def hts_to_etrago(scenario): sources = config.datasets()["etrago_heat"]["sources"] targets = config.datasets()["etrago_heat"]["targets"] - scenario = "eGon2035" carriers = ["central_heat", "rural_heat", "rural_gas_boiler"] + if "status" in scenario: + carriers = ["central_heat", "rural_heat"] + for carrier in carriers: if carrier == "central_heat": - # Map heat buses to district heating id and area_id # interlinking bus_id and area_id bus_area = db.select_dataframe( @@ -92,7 +92,7 @@ def hts_to_etrago(): bus_ts.loc[:, "bus_id"] = bus_ts.loc[:, "heat_bus_id"] else: - efficiency_gas_boiler = get_sector_parameters("heat", "eGon2035")[ + efficiency_gas_boiler = get_sector_parameters("heat", scenario)[ "efficiency" ]["rural_gas_boiler"] @@ -212,6 +212,18 @@ def hts_to_etrago(): ) +def demand(): + """Insert demand timeseries for heat into eTraGo tables + + Returns + ------- + None. + + """ + for scenario in config.settings()["egon-data"]["--scenarios"]: + hts_to_etrago(scenario) + + class HtsEtragoTable(Dataset): """ Collect heat demand time series for the eTraGo tool @@ -242,5 +254,5 @@ def __init__(self, dependencies): name=self.name, version=self.version, dependencies=dependencies, - tasks=(hts_to_etrago,), + tasks=(demand,), ) diff --git a/src/egon/data/datasets/heat_etrago/power_to_heat.py b/src/egon/data/datasets/heat_etrago/power_to_heat.py index d32f09459..8d7cc3bb5 100644 --- a/src/egon/data/datasets/heat_etrago/power_to_heat.py +++ b/src/egon/data/datasets/heat_etrago/power_to_heat.py @@ -9,13 +9,13 @@ from egon.data.datasets.scenario_parameters import get_sector_parameters -def insert_individual_power_to_heat(scenario="eGon2035"): +def insert_individual_power_to_heat(scenario): """Insert power to heat into database Parameters ---------- scenario : str, optional - Name of the scenario The default is 'eGon2035'. + Name of the scenario. Returns ------- @@ -90,7 +90,7 @@ def insert_individual_power_to_heat(scenario="eGon2035"): heat_pumps["voltage_level"] = 7 # Set marginal_cost - heat_pumps["marginal_cost"] = get_sector_parameters("heat", "eGon2035")[ + heat_pumps["marginal_cost"] = get_sector_parameters("heat", scenario)[ "marginal_cost" ]["rural_heat_pump"] @@ -99,17 +99,17 @@ def insert_individual_power_to_heat(scenario="eGon2035"): heat_pumps, carrier="rural_heat_pump", multiple_per_mv_grid=False, - scenario="eGon2035", + scenario=scenario, ) -def insert_central_power_to_heat(scenario="eGon2035"): +def insert_central_power_to_heat(scenario): """Insert power to heat in district heating areas into database Parameters ---------- - scenario : str, optional - Name of the scenario The default is 'eGon2035'. + scenario : str + Name of the scenario. Returns ------- @@ -182,7 +182,7 @@ def insert_central_power_to_heat(scenario="eGon2035"): # Set marginal_cost central_heat_pumps["marginal_cost"] = get_sector_parameters( - "heat", "eGon2035" + "heat", scenario )["marginal_cost"]["central_heat_pump"] # Insert heatpumps in mv and below @@ -190,14 +190,16 @@ def insert_central_power_to_heat(scenario="eGon2035"): insert_power_to_heat_per_level( central_heat_pumps[central_heat_pumps.voltage_level > 3], multiple_per_mv_grid=False, - scenario="eGon2035", + carrier = "central_heat_pump", + scenario=scenario, ) # Insert heat pumps in hv grid # (as many hvmv substations as intersect with district heating grid) insert_power_to_heat_per_level( central_heat_pumps[central_heat_pumps.voltage_level < 3], multiple_per_mv_grid=True, - scenario="eGon2035", + carrier = "central_heat_pump", + scenario=scenario, ) # Delete existing entries @@ -242,7 +244,7 @@ def insert_central_power_to_heat(scenario="eGon2035"): # Set efficiency central_resistive_heater["efficiency"] = get_sector_parameters( - "heat", "eGon2035" + "heat", scenario )["efficiency"]["central_resistive_heater"] # Insert heatpumps in mv and below @@ -261,7 +263,7 @@ def insert_central_power_to_heat(scenario="eGon2035"): ], multiple_per_mv_grid=False, carrier="central_resistive_heater", - scenario="eGon2035", + scenario=scenario, ) # Insert heat pumps in hv grid # (as many hvmv substations as intersect with district heating grid) @@ -269,15 +271,15 @@ def insert_central_power_to_heat(scenario="eGon2035"): central_resistive_heater[central_resistive_heater.voltage_level < 3], multiple_per_mv_grid=True, carrier="central_resistive_heater", - scenario="eGon2035", + scenario=scenario, ) def insert_power_to_heat_per_level( heat_pumps, multiple_per_mv_grid, - carrier="central_heat_pump", - scenario="eGon2035", + carrier, + scenario, ): """Insert power to heat plants per grid level @@ -287,8 +289,8 @@ def insert_power_to_heat_per_level( Heat pumps in selected grid level multiple_per_mv_grid : boolean Choose if one district heating areas is supplied by one hvmv substation - scenario : str, optional - Name of the scenario The default is 'eGon2035'. + scenario : str + Name of the scenario. Returns ------- @@ -300,7 +302,8 @@ def insert_power_to_heat_per_level( if "central" in carrier: # Calculate heat pumps per electrical bus - gdf = assign_electrical_bus(heat_pumps, carrier, multiple_per_mv_grid) + gdf = assign_electrical_bus( + heat_pumps, carrier, scenario, multiple_per_mv_grid) else: gdf = heat_pumps.copy() @@ -433,7 +436,7 @@ def assign_voltage_level(heat_pumps, carrier="heat_pump"): return heat_pumps -def assign_electrical_bus(heat_pumps, carrier, multiple_per_mv_grid=False): +def assign_electrical_bus(heat_pumps, carrier, scenario, multiple_per_mv_grid=False): """Calculates heat pumps per electrical bus Parameters @@ -464,8 +467,8 @@ def assign_electrical_bus(heat_pumps, carrier, multiple_per_mv_grid=False): {sources['district_heating_areas']['table']} ON ST_Transform(ST_Centroid(geom_polygon), 4326) = geom WHERE carrier = 'central_heat' - AND scenario='eGon2035' - AND scn_name = 'eGon2035' + AND scenario='{scenario}' + AND scn_name = '{scenario}' """, index_col="id", ) @@ -495,8 +498,8 @@ def assign_electrical_bus(heat_pumps, carrier, multiple_per_mv_grid=False): JOIN society.destatis_zensus_population_per_ha ON society.destatis_zensus_population_per_ha.id = a.zensus_population_id - WHERE a.scenario = 'eGon2035' - AND b.scenario = 'eGon2035' + WHERE a.scenario = '{scenario}' + AND b.scenario = '{scenario}' GROUP BY (area_id, a.zensus_population_id, geom_point) """, epsg=4326, diff --git a/src/egon/data/datasets/heat_supply/__init__.py b/src/egon/data/datasets/heat_supply/__init__.py index dfaf2c797..6bc14ae34 100644 --- a/src/egon/data/datasets/heat_supply/__init__.py +++ b/src/egon/data/datasets/heat_supply/__init__.py @@ -2,22 +2,35 @@ """ -from egon.data import db, config +import datetime +import json +import time +from geoalchemy2.types import Geometry +from sqlalchemy import Column, Float, ForeignKey, Integer, String +from sqlalchemy.ext.declarative import declarative_base +import pandas as pd + +from egon.data import config, db +from egon.data.datasets import Dataset +from egon.data.datasets.district_heating_areas import EgonDistrictHeatingAreas from egon.data.datasets.heat_supply.district_heating import ( - cascade_heat_supply, backup_gas_boilers, backup_resistive_heaters, + cascade_heat_supply, ) +from egon.data.datasets.heat_supply.geothermal import potential_germany from egon.data.datasets.heat_supply.individual_heating import ( cascade_heat_supply_indiv, ) -from egon.data.datasets.heat_supply.geothermal import potential_germany -from egon.data.datasets.district_heating_areas import EgonDistrictHeatingAreas -from sqlalchemy import Column, String, Float, Integer, ForeignKey -from sqlalchemy.ext.declarative import declarative_base -from geoalchemy2.types import Geometry -from egon.data.datasets import Dataset +from egon.data.metadata import ( + context, + generate_resource_fields_from_sqla_model, + license_ccby, + license_egon_data_odbl, + meta_metadata, + sources, +) # Will later be imported from another file. Base = declarative_base() @@ -82,53 +95,47 @@ def district_heating(): """ ) - supply_2035 = cascade_heat_supply("eGon2035", plotting=False) + for scenario in config.settings()["egon-data"]["--scenarios"]: + supply = cascade_heat_supply(scenario, plotting=False) - supply_2035["scenario"] = "eGon2035" + supply["scenario"] = scenario - supply_2035.to_postgis( - targets["district_heating_supply"]["table"], - schema=targets["district_heating_supply"]["schema"], - con=db.engine(), - if_exists="append", - ) - - # Compare target value with sum of distributed heat supply - df_check = db.select_dataframe( - f""" - SELECT a.carrier, - (SUM(a.capacity) - b.capacity) / SUM(a.capacity) as deviation - FROM {targets['district_heating_supply']['schema']}. - {targets['district_heating_supply']['table']} a, - {sources['scenario_capacities']['schema']}. - {sources['scenario_capacities']['table']} b - WHERE a.scenario = 'eGon2035' - AND b.scenario_name = 'eGon2035' - AND b.carrier = CONCAT('urban_central_', a.carrier) - GROUP BY (a.carrier, b.capacity); - """ - ) - # If the deviation is > 1%, throw an error - assert ( - df_check.deviation.abs().max() < 1 - ), f"""Unexpected deviation between target value and distributed - heat supply: {df_check} - """ - - # Add gas boilers as conventional backup capacities - backup = backup_gas_boilers("eGon2035") - - backup.to_postgis( - targets["district_heating_supply"]["table"], - schema=targets["district_heating_supply"]["schema"], - con=db.engine(), - if_exists="append", - ) + supply.to_postgis( + targets["district_heating_supply"]["table"], + schema=targets["district_heating_supply"]["schema"], + con=db.engine(), + if_exists="append", + ) - backup_rh = backup_resistive_heaters("eGon2035") - if not backup_rh.empty: - backup_rh.to_postgis( + # Do not check data for status quo as is it not listed in the table + if "status" not in scenario: + # Compare target value with sum of distributed heat supply + df_check = db.select_dataframe( + f""" + SELECT a.carrier, + (SUM(a.capacity) - b.capacity) / SUM(a.capacity) as deviation + FROM {targets['district_heating_supply']['schema']}. + {targets['district_heating_supply']['table']} a, + {sources['scenario_capacities']['schema']}. + {sources['scenario_capacities']['table']} b + WHERE a.scenario = '{scenario}' + AND b.scenario_name = '{scenario}' + AND b.carrier = CONCAT('urban_central_', a.carrier) + GROUP BY (a.carrier, b.capacity); + """ + ) + # If the deviation is > 1%, throw an error + assert ( + df_check.deviation.abs().max() < 1 + ), f"""Unexpected deviation between target value and distributed + heat supply: {df_check} + """ + + # Add gas boilers as conventional backup capacities + backup = backup_gas_boilers(scenario) + + backup.to_postgis( targets["district_heating_supply"]["table"], schema=targets["district_heating_supply"]["schema"], con=db.engine(), @@ -136,6 +143,19 @@ def district_heating(): ) + # Insert resistive heaters which are not available in status quo + if "status" not in scenario: + backup_rh = backup_resistive_heaters(scenario) + + if not backup_rh.empty: + backup_rh.to_postgis( + targets["district_heating_supply"]["table"], + schema=targets["district_heating_supply"]["schema"], + con=db.engine(), + if_exists="append", + ) + + def individual_heating(): """Insert supply for individual heating @@ -146,24 +166,200 @@ def individual_heating(): """ targets = config.datasets()["heat_supply"]["targets"] - db.execute_sql( - f""" - DELETE FROM {targets['individual_heating_supply']['schema']}. - {targets['individual_heating_supply']['table']} - """ + for scenario in config.settings()["egon-data"]["--scenarios"]: + db.execute_sql( + f""" + DELETE FROM {targets['individual_heating_supply']['schema']}. + {targets['individual_heating_supply']['table']} + WHERE scenario = '{scenario}' + """ + ) + if scenario == "eGon2035": + distribution_level = "federal_states" + else: + distribution_level = "national" + + supply = cascade_heat_supply_indiv( + scenario, distribution_level=distribution_level, plotting=False + ) + + supply["scenario"] = scenario + + supply.to_postgis( + targets["individual_heating_supply"]["table"], + schema=targets["individual_heating_supply"]["schema"], + con=db.engine(), + if_exists="append", + ) + + +def metadata(): + """Write metadata for heat supply tables + + Returns + ------- + None. + + """ + + fields = generate_resource_fields_from_sqla_model( + EgonDistrictHeatingSupply ) - supply_2035 = cascade_heat_supply_indiv( - "eGon2035", distribution_level="federal_states", plotting=False + fields_df = pd.DataFrame(data=fields).set_index("name") + fields_df.loc["index", "description"] = "Unique identifyer" + fields_df.loc[ + "district_heating_id", "description" + ] = "Index of the corresponding district heating grid" + fields_df.loc["carrier", "description"] = "Name of energy carrier" + fields_df.loc[ + "category", "description" + ] = "Size-category of district heating grid" + fields_df.loc["capacity", "description"] = "Installed heating capacity" + fields_df.loc[ + "geometry", "description" + ] = "Location of thermal power plant" + fields_df.loc["scenario", "description"] = "Name of corresponing scenario" + + fields_df.loc["capacity", "unit"] = "MW_th" + fields_df.unit.fillna("none", inplace=True) + + fields = fields_df.reset_index().to_dict(orient="records") + + meta_district = { + "name": "supply.egon_district_heating", + "title": "eGon heat supply for district heating grids", + "id": "WILL_BE_SET_AT_PUBLICATION", + "description": "Heat supply technologies for district heating grids", + "language": ["EN"], + "publicationDate": datetime.date.today().isoformat(), + "context": context(), + "spatial": { + "location": None, + "extent": "Germany", + "resolution": None, + }, + "sources": [ + sources()["era5"], + sources()["vg250"], + sources()["egon-data"], + sources()["egon-data_bundle"], + sources()["openstreetmap"], + sources()["mastr"], + sources()["peta"], + ], + "licenses": [license_egon_data_odbl()], + "contributors": [ + { + "title": "Clara Büttner", + "email": "http://github.com/ClaraBuettner", + "date": time.strftime("%Y-%m-%d"), + "object": None, + "comment": "Imported data", + }, + ], + "resources": [ + { + "profile": "tabular-data-resource", + "name": "supply.egon_district_heating", + "path": None, + "format": "PostgreSQL", + "encoding": "UTF-8", + "schema": { + "fields": fields, + "primaryKey": ["index"], + "foreignKeys": [], + }, + "dialect": {"delimiter": None, "decimalSeparator": "."}, + } + ], + "metaMetadata": meta_metadata(), + } + + # Add metadata as a comment to the table + db.submit_comment( + "'" + json.dumps(meta_district) + "'", + EgonDistrictHeatingSupply.__table__.schema, + EgonDistrictHeatingSupply.__table__.name, ) - supply_2035["scenario"] = "eGon2035" + fields = generate_resource_fields_from_sqla_model( + EgonIndividualHeatingSupply + ) - supply_2035.to_postgis( - targets["individual_heating_supply"]["table"], - schema=targets["individual_heating_supply"]["schema"], - con=db.engine(), - if_exists="append", + fields_df = pd.DataFrame(data=fields).set_index("name") + fields_df.loc["index", "description"] = "Unique identifyer" + fields_df.loc[ + "mv_grid_id", "description" + ] = "Index of the corresponding mv grid district" + fields_df.loc["carrier", "description"] = "Name of energy carrier" + fields_df.loc["category", "description"] = "Size-category" + fields_df.loc["capacity", "description"] = "Installed heating capacity" + fields_df.loc[ + "geometry", "description" + ] = "Location of thermal power plant" + fields_df.loc["scenario", "description"] = "Name of corresponing scenario" + + fields_df.loc["capacity", "unit"] = "MW_th" + fields_df.unit.fillna("none", inplace=True) + + fields = fields_df.reset_index().to_dict(orient="records") + + meta_district = { + "name": "supply.egon_individual_heating", + "title": "eGon heat supply for individual supplied buildings", + "id": "WILL_BE_SET_AT_PUBLICATION", + "description": "Heat supply technologies for individual supplied buildings", + "language": ["EN"], + "publicationDate": datetime.date.today().isoformat(), + "context": context(), + "spatial": { + "location": None, + "extent": "Germany", + "resolution": None, + }, + "sources": [ + sources()["era5"], + sources()["vg250"], + sources()["egon-data"], + sources()["egon-data_bundle"], + sources()["openstreetmap"], + sources()["mastr"], + sources()["peta"], + ], + "licenses": [license_egon_data_odbl()], + "contributors": [ + { + "title": "Clara Büttner", + "email": "http://github.com/ClaraBuettner", + "date": time.strftime("%Y-%m-%d"), + "object": None, + "comment": "Imported data", + }, + ], + "resources": [ + { + "profile": "tabular-data-resource", + "name": "supply.egon_individual_heating", + "path": None, + "format": "PostgreSQL", + "encoding": "UTF-8", + "schema": { + "fields": fields, + "primaryKey": ["index"], + "foreignKeys": [], + }, + "dialect": {"delimiter": None, "decimalSeparator": "."}, + } + ], + "metaMetadata": meta_metadata(), + } + + # Add metadata as a comment to the table + db.submit_comment( + "'" + json.dumps(meta_district) + "'", + EgonIndividualHeatingSupply.__table__.schema, + EgonIndividualHeatingSupply.__table__.name, ) @@ -195,7 +391,7 @@ class HeatSupply(Dataset): #: name: str = "HeatSupply" #: - version: str = "0.0.8" + version: str = "0.0.9" def __init__(self, dependencies): super().__init__( @@ -209,5 +405,6 @@ def __init__(self, dependencies): individual_heating, potential_germany, }, + metadata, ), ) diff --git a/src/egon/data/datasets/heat_supply/district_heating.py b/src/egon/data/datasets/heat_supply/district_heating.py index a2d2501e6..15d0f3dfe 100644 --- a/src/egon/data/datasets/heat_supply/district_heating.py +++ b/src/egon/data/datasets/heat_supply/district_heating.py @@ -161,9 +161,13 @@ def select_district_heating_areas(scenario): def cascade_per_technology( - areas, technologies, capacity_per_category, size_dh, max_geothermal_costs=2 + scenario, + areas, + technologies, + capacity_per_category, + size_dh, + max_geothermal_costs=2, ): - """Add plants of one technology suppliing district heating Parameters @@ -196,7 +200,6 @@ def cascade_per_technology( # Assign CHP plants inside district heating area # TODO: This has to be updaten when all chp plants are available! if tech.index == "CHP": - # Select chp plants from database gdf_chp = db.select_geodataframe( f"""SELECT a.geom, th_capacity as capacity, c.area_id @@ -206,8 +209,8 @@ def cascade_per_technology( {sources['district_heating_areas']['table']} c WHERE a.district_heating = True AND a.district_heating_area_id = c.area_id - AND a.scenario = 'eGon2035' - AND c.scenario = 'eGon2035' + AND a.scenario = '{scenario}' + AND c.scenario = '{scenario}' """ ) @@ -227,7 +230,6 @@ def cascade_per_technology( "heat_pump", "geo_thermal", ]: - if tech.index == "geo_thermal": # Select areas with geothermal potential considering costs gdf_geothermal = calc_geothermal_costs(max_geothermal_costs) @@ -305,26 +307,38 @@ def cascade_heat_supply(scenario, plotting=True): district_heating_areas = select_district_heating_areas(scenario) # Select technolgies per district heating size - map_dh_technologies = { - "small": [ - "CHP", - "solar_thermal_collector", - "heat_pump", - "resistive_heater", - ], - "medium": [ - "CHP", - "solar_thermal_collector", - "heat_pump", - "resistive_heater", - ], - "large": ["CHP", "geo_thermal", "heat_pump", "resistive_heater"], - } + if "status" not in scenario: + map_dh_technologies = { + "small": [ + "CHP", + "solar_thermal_collector", + "heat_pump", + "resistive_heater", + ], + "medium": [ + "CHP", + "solar_thermal_collector", + "heat_pump", + "resistive_heater", + ], + "large": ["CHP", "geo_thermal", "heat_pump", "resistive_heater"], + } - # Assign capacities per district heating category - capacity_per_category = capacity_per_district_heating_category( - district_heating_areas, scenario - ) + # Assign capacities per district heating category + capacity_per_category = capacity_per_district_heating_category( + district_heating_areas, scenario + ) + else: + map_dh_technologies = { + "small": [ + "CHP", + ], + "medium": [ + "CHP", + ], + "large": ["CHP"], + } + capacity_per_category = pd.DataFrame() # Initalize Dataframe for results resulting_capacities = pd.DataFrame( @@ -335,7 +349,6 @@ def cascade_heat_supply(scenario, plotting=True): technology_data = set_technology_data() for size_dh in ["small", "medium", "large"]: - # Select areas in size-category areas = district_heating_areas[ district_heating_areas.category == size_dh @@ -350,13 +363,12 @@ def cascade_heat_supply(scenario, plotting=True): # Assign new supply technologies to district heating areas # as long as the demand is not covered and there are technologies left while (len(technologies) > 0) and (len(areas) > 0): - areas, technologies, append_df = cascade_per_technology( - areas, technologies, capacity_per_category, size_dh + scenario, areas, technologies, capacity_per_category, size_dh ) - resulting_capacities = resulting_capacities.append( - append_df, ignore_index=True + resulting_capacities = pd.concat( + [resulting_capacities, append_df], ignore_index=True ) # Plot results per district heating area @@ -392,7 +404,7 @@ def backup_gas_boilers(scenario): return gpd.GeoDataFrame( data={ "district_heating_id": district_heating_areas.index, - "capacity": district_heating_areas.demand.div(8000), + "capacity": district_heating_areas.demand.div(100), "carrier": "gas_boiler", "category": district_heating_areas.category, "geometry": district_heating_areas.geom.centroid, @@ -460,7 +472,6 @@ def backup_resistive_heaters(scenario): def plot_heat_supply(resulting_capacities): - from matplotlib import pyplot as plt district_heating_areas = select_district_heating_areas("eGon2035") diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index 608a73a7e..ea1148f57 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -184,7 +184,7 @@ import os import random -from airflow.operators.python_operator import PythonOperator +from airflow.operators.python import PythonOperator from psycopg2.extensions import AsIs, register_adapter from sqlalchemy import ARRAY, REAL, Column, Integer, String from sqlalchemy.ext.declarative import declarative_base @@ -194,7 +194,7 @@ import saio from egon.data import config, db, logger -from egon.data.datasets import Dataset +from egon.data.datasets import Dataset, wrapped_partial from egon.data.datasets.district_heating_areas import ( MapZensusDistrictHeatingAreas, ) @@ -208,7 +208,7 @@ write_table_to_postgres, ) from egon.data.datasets.emobility.motorized_individual_travel.helpers import ( - reduce_mem_usage + reduce_mem_usage, ) from egon.data.datasets.heat_demand import EgonPetaHeat from egon.data.datasets.heat_demand_timeseries.daily import ( @@ -264,6 +264,7 @@ def dyn_parallel_tasks_pypsa_eur_sec(): ) tasks = set() + for i in range(parallel_tasks): tasks.add( PythonOperator( @@ -295,6 +296,85 @@ def dyn_parallel_tasks_pypsa_eur_sec(): ) +class HeatPumpsStatusQuo(Dataset): + def __init__(self, dependencies): + def dyn_parallel_tasks_status_quo(scenario): + """Dynamically generate tasks + + The goal is to speed up tasks by parallelising bulks of mvgds. + + The number of parallel tasks is defined via parameter + `parallel_tasks` in the dataset config `datasets.yml`. + + Returns + ------- + set of airflow.PythonOperators + The tasks. Each element is of + :func:`egon.data.datasets.heat_supply.individual_heating. + determine_hp_cap_peak_load_mvgd_ts_status_quo` + """ + parallel_tasks = config.datasets()["demand_timeseries_mvgd"].get( + "parallel_tasks", 1 + ) + + tasks = set() + + for i in range(parallel_tasks): + tasks.add( + PythonOperator( + task_id=( + "individual_heating." + f"determine-hp-capacity-{scenario}-" + f"mvgd-bulk{i}" + ), + python_callable=split_mvgds_into_bulks, + op_kwargs={ + "n": i, + "max_n": parallel_tasks, + "scenario": scenario, + "func": determine_hp_cap_peak_load_mvgd_ts_status_quo, + }, + ) + ) + return tasks + + tasks = () + + for scenario in config.settings()["egon-data"]["--scenarios"]: + if "status" in scenario: + postfix = f"_{scenario[-4:]}" + + tasks += ( + wrapped_partial( + delete_heat_peak_loads_status_quo, + scenario=scenario, + postfix=postfix, + ), + wrapped_partial( + delete_hp_capacity_status_quo, + scenario=scenario, + postfix=postfix, + ), + wrapped_partial( + delete_mvgd_ts_status_quo, + scenario=scenario, + postfix=postfix, + ), + ) + + tasks += ( + {*dyn_parallel_tasks_status_quo(scenario)}, + ) + + + super().__init__( + name="HeatPumpsStatusQuo", + version="0.0.4", + dependencies=dependencies, + tasks=tasks, + ) + + class HeatPumps2035(Dataset): def __init__(self, dependencies): def dyn_parallel_tasks_2035(): @@ -315,7 +395,9 @@ def dyn_parallel_tasks_2035(): parallel_tasks = config.datasets()["demand_timeseries_mvgd"].get( "parallel_tasks", 1 ) + tasks = set() + for i in range(parallel_tasks): tasks.add( PythonOperator( @@ -385,7 +467,6 @@ def cascade_per_technology( distribution_level, max_size_individual_chp=0.05, ): - """Add plants for individual heat. Currently only on mv grid district level. @@ -415,7 +496,6 @@ def cascade_per_technology( # Distribute heat pumps linear to remaining demand. if tech.index == "heat_pump": - if distribution_level == "federal_state": # Select target values per federal state target = db.select_dataframe( @@ -467,7 +547,6 @@ def cascade_per_technology( ) elif tech.index == "gas_boiler": - append_df = pd.DataFrame( data={ "capacity": heat_per_mv.remaining_demand.div( @@ -557,11 +636,20 @@ def cascade_heat_supply_indiv(scenario, distribution_level, plotting=True): # Set technology data according to # http://www.wbzu.de/seminare/infopool/infopool-bhkw # TODO: Add gas boilers and solar themal (eGon100RE) - technologies = pd.DataFrame( - index=["heat_pump", "gas_boiler"], - columns=["estimated_flh", "priority"], - data={"estimated_flh": [4000, 8000], "priority": [2, 1]}, - ) + if scenario == "eGon2035": + technologies = pd.DataFrame( + index=["heat_pump", "gas_boiler"], + columns=["estimated_flh", "priority"], + data={"estimated_flh": [4000, 8000], "priority": [2, 1]}, + ) + elif "status" in scenario: + technologies = pd.DataFrame( + index=["heat_pump"], + columns=["estimated_flh", "priority"], + data={"estimated_flh": [4000], "priority": [2]}, + ) + else: + raise ValueError(f"{scenario=} is not valid.") # In the beginning, the remaining demand equals demand heat_per_mv["remaining_demand"] = heat_per_mv["demand"] @@ -573,8 +661,8 @@ def cascade_heat_supply_indiv(scenario, distribution_level, plotting=True): heat_per_mv, technologies, scenario, distribution_level ) # Collect resulting capacities - resulting_capacities = resulting_capacities.append( - append_df, ignore_index=True + resulting_capacities = pd.concat( + [resulting_capacities, append_df], ignore_index=True ) if plotting: @@ -816,8 +904,10 @@ def calc_residential_heat_profiles_per_mvgd(mvgd, scenario): left=df_peta_demand, right=df_profiles_ids, on="zensus_population_id" ) - df_profile_merge.demand = df_profile_merge.demand.div(df_profile_merge.buildings) - df_profile_merge.drop('buildings', axis='columns', inplace=True) + df_profile_merge.demand = df_profile_merge.demand.div( + df_profile_merge.buildings + ) + df_profile_merge.drop("buildings", axis="columns", inplace=True) # Merge daily demand to daily profile ids by zensus_population_id and day df_profile_merge = pd.merge( @@ -826,8 +916,9 @@ def calc_residential_heat_profiles_per_mvgd(mvgd, scenario): on=["zensus_population_id", "day_of_year"], ) df_profile_merge.demand = df_profile_merge.demand.mul( - df_profile_merge.daily_demand_share) - df_profile_merge.drop('daily_demand_share', axis='columns', inplace=True) + df_profile_merge.daily_demand_share + ) + df_profile_merge.drop("daily_demand_share", axis="columns", inplace=True) df_profile_merge = reduce_mem_usage(df_profile_merge) # Merge daily profiles by profile id @@ -840,10 +931,13 @@ def calc_residential_heat_profiles_per_mvgd(mvgd, scenario): df_profile_merge = reduce_mem_usage(df_profile_merge) df_profile_merge.demand = df_profile_merge.demand.mul( - df_profile_merge.idp.astype(float)) - df_profile_merge.drop('idp', axis='columns', inplace=True) + df_profile_merge.idp.astype(float) + ) + df_profile_merge.drop("idp", axis="columns", inplace=True) - df_profile_merge.rename({'demand': 'demand_ts'}, axis='columns', inplace=True) + df_profile_merge.rename( + {"demand": "demand_ts"}, axis="columns", inplace=True + ) df_profile_merge = reduce_mem_usage(df_profile_merge) @@ -851,7 +945,6 @@ def calc_residential_heat_profiles_per_mvgd(mvgd, scenario): def plot_heat_supply(resulting_capacities): - from matplotlib import pyplot as plt mv_grids = db.select_geodataframe( @@ -1084,7 +1177,7 @@ def get_buildings_with_decentral_heat_demand_in_mv_grid(mvgd, scenario): ) # merge residential and CTS buildings - buildings_decentral_heating = buildings_decentral_heating_res.append( + buildings_decentral_heating = buildings_decentral_heating_res.union( buildings_decentral_heating_cts ).unique() @@ -1269,7 +1362,7 @@ def determine_buildings_with_hp_in_mv_grid( random.seed(db.credentials()["--random-seed"]) new_hp_building = random.choice(possible_buildings) # add new building to building with HP - buildings_with_hp = buildings_with_hp.append( + buildings_with_hp = buildings_with_hp.union( pd.Index([new_hp_building]) ) # determine if there are still possible buildings @@ -1357,8 +1450,8 @@ def determine_min_hp_cap_buildings_pypsa_eur_sec( return 0.0 -def determine_hp_cap_buildings_eGon2035_per_mvgd( - mv_grid_id, peak_heat_demand, building_ids +def determine_hp_cap_buildings_pvbased_per_mvgd( + scenario, mv_grid_id, peak_heat_demand, building_ids ): """ Determines which buildings in the MV grid will have a HP (buildings with PV @@ -1378,9 +1471,7 @@ def determine_hp_cap_buildings_eGon2035_per_mvgd( """ - hp_cap_grid = get_total_heat_pump_capacity_of_mv_grid( - "eGon2035", mv_grid_id - ) + hp_cap_grid = get_total_heat_pump_capacity_of_mv_grid(scenario, mv_grid_id) if len(building_ids) > 0 and hp_cap_grid > 0.0: peak_heat_demand = peak_heat_demand.loc[building_ids] @@ -1425,7 +1516,6 @@ def determine_hp_cap_buildings_eGon100RE_per_mvgd(mv_grid_id): ) if hp_cap_grid > 0.0: - # get buildings with decentral heating systems building_ids = get_buildings_with_decentral_heat_demand_in_mv_grid( mv_grid_id, scenario="eGon100RE" @@ -1486,7 +1576,6 @@ def determine_hp_cap_buildings_eGon100RE(): ) for mvgd_id in mvgd_ids: - logger.info(f"MVGD={mvgd_id} | Start") hp_cap_per_building_100RE = ( @@ -1638,9 +1727,7 @@ def export_min_cap_to_csv(df_hp_min_cap_mv_grid_pypsa_eur_sec): os.mkdir(folder) if not file.is_file(): logger.info(f"Create {file}") - df_hp_min_cap_mv_grid_pypsa_eur_sec.to_csv( - file, mode="w", header=True - ) + df_hp_min_cap_mv_grid_pypsa_eur_sec.to_csv(file, mode="w", header=True) else: df_hp_min_cap_mv_grid_pypsa_eur_sec.to_csv( file, mode="a", header=False @@ -1713,7 +1800,6 @@ def determine_hp_cap_peak_load_mvgd_ts_2035(mvgd_ids): df_heat_mvgd_ts_db = pd.DataFrame() for mvgd in mvgd_ids: - logger.info(f"MVGD={mvgd} | Start") # ############# aggregate residential and CTS demand profiles ##### @@ -1743,12 +1829,11 @@ def determine_hp_cap_peak_load_mvgd_ts_2035(mvgd_ids): buildings_decentral_heating, peak_load_2035 ) - hp_cap_per_building_2035 = ( - determine_hp_cap_buildings_eGon2035_per_mvgd( - mvgd, - peak_load_2035, - buildings_decentral_heating, - ) + hp_cap_per_building_2035 = determine_hp_cap_buildings_pvbased_per_mvgd( + "eGon2035", + mvgd, + peak_load_2035, + buildings_decentral_heating, ) buildings_gas_2035 = pd.Index(buildings_decentral_heating).drop( hp_cap_per_building_2035.index @@ -1810,13 +1895,18 @@ def determine_hp_cap_peak_load_mvgd_ts_2035(mvgd_ids): df_hp_cap_per_building_2035_db.duplicated("building_id", keep=False) ] - logger.info( - f"Dropped duplicated buildings: " - f"{duplicates.loc[:,['building_id', 'hp_capacity']]}" - ) + if not duplicates.empty: + logger.info( + f"Dropped duplicated buildings: " + f"{duplicates.loc[:,['building_id', 'hp_capacity']]}" + ) df_hp_cap_per_building_2035_db.drop_duplicates("building_id", inplace=True) + df_hp_cap_per_building_2035_db.building_id = ( + df_hp_cap_per_building_2035_db.building_id.astype(int) + ) + write_table_to_postgres( df_hp_cap_per_building_2035_db, EgonHpCapacityBuildings, @@ -1824,6 +1914,134 @@ def determine_hp_cap_peak_load_mvgd_ts_2035(mvgd_ids): ) +def determine_hp_cap_peak_load_mvgd_ts_status_quo(mvgd_ids, scenario): + """ + Main function to determine HP capacity per building in status quo scenario. + Further, creates heat demand time series for all buildings with heat pumps + in MV grid, as well as for all buildings with gas boilers, used in eTraGo. + + Parameters + ----------- + mvgd_ids : list(int) + List of MV grid IDs to determine data for. + + """ + + # ========== Register np datatypes with SQLA ========== + register_adapter(np.float64, adapt_numpy_float64) + register_adapter(np.int64, adapt_numpy_int64) + # ===================================================== + + df_peak_loads_db = pd.DataFrame() + df_hp_cap_per_building_status_quo_db = pd.DataFrame() + df_heat_mvgd_ts_db = pd.DataFrame() + + for mvgd in mvgd_ids: + logger.info(f"MVGD={mvgd} | Start") + + # ############# aggregate residential and CTS demand profiles ##### + + df_heat_ts = aggregate_residential_and_cts_profiles( + mvgd, scenario=scenario + ) + + # ##################### determine peak loads ################### + logger.info(f"MVGD={mvgd} | Determine peak loads.") + + peak_load_status_quo = df_heat_ts.max().rename(scenario) + + # ######## determine HP capacity per building ######### + logger.info(f"MVGD={mvgd} | Determine HP capacities.") + + buildings_decentral_heating = ( + get_buildings_with_decentral_heat_demand_in_mv_grid( + mvgd, scenario=scenario + ) + ) + + # Reduce list of decentral heating if no Peak load available + # TODO maybe remove after succesfull DE run + # Might be fixed in #990 + buildings_decentral_heating = catch_missing_buidings( + buildings_decentral_heating, peak_load_status_quo + ) + + hp_cap_per_building_status_quo = determine_hp_cap_buildings_pvbased_per_mvgd( + scenario, + mvgd, + peak_load_status_quo, + buildings_decentral_heating, + ) + + # ################ aggregated heat profiles ################### + logger.info(f"MVGD={mvgd} | Aggregate heat profiles.") + + df_mvgd_ts_status_quo_hp = df_heat_ts.loc[ + :, + hp_cap_per_building_status_quo.index, + ].sum(axis=1) + + df_heat_mvgd_ts = pd.DataFrame( + data={ + "carrier": "heat_pump", + "bus_id": mvgd, + "scenario": scenario, + "dist_aggregated_mw": [df_mvgd_ts_status_quo_hp.to_list()], + } + ) + + # ################ collect results ################## + logger.info(f"MVGD={mvgd} | Collect results.") + + df_peak_loads_db = pd.concat( + [df_peak_loads_db, peak_load_status_quo.reset_index()], + axis=0, + ignore_index=True, + ) + + df_heat_mvgd_ts_db = pd.concat( + [df_heat_mvgd_ts_db, df_heat_mvgd_ts], axis=0, ignore_index=True + ) + + df_hp_cap_per_building_status_quo_db = pd.concat( + [ + df_hp_cap_per_building_status_quo_db, + hp_cap_per_building_status_quo.reset_index(), + ], + axis=0, + ) + + # ################ export to db ####################### + logger.info(f"MVGD={min(mvgd_ids)} : {max(mvgd_ids)} | Write data to db.") + + export_to_db(df_peak_loads_db, df_heat_mvgd_ts_db, drop=False) + + df_hp_cap_per_building_status_quo_db["scenario"] = scenario + + # TODO debug duplicated building_ids + duplicates = df_hp_cap_per_building_status_quo_db.loc[ + df_hp_cap_per_building_status_quo_db.duplicated("building_id", keep=False) + ] + + if not duplicates.empty: + logger.info( + f"Dropped duplicated buildings: " + f"{duplicates.loc[:,['building_id', 'hp_capacity']]}" + ) + + df_hp_cap_per_building_status_quo_db.drop_duplicates("building_id", inplace=True) + + df_hp_cap_per_building_status_quo_db.building_id = ( + df_hp_cap_per_building_status_quo_db.building_id.astype(int) + ) + + write_table_to_postgres( + df_hp_cap_per_building_status_quo_db, + EgonHpCapacityBuildings, + drop=False, + ) + + def determine_hp_cap_peak_load_mvgd_ts_pypsa_eur_sec(mvgd_ids): """ Main function to determine minimum required HP capacity in MV for @@ -1847,7 +2065,6 @@ def determine_hp_cap_peak_load_mvgd_ts_pypsa_eur_sec(mvgd_ids): df_hp_min_cap_mv_grid_pypsa_eur_sec = pd.Series(dtype="float64") for mvgd in mvgd_ids: - logger.info(f"MVGD={mvgd} | Start") # ############# aggregate residential and CTS demand profiles ##### @@ -1930,7 +2147,7 @@ def determine_hp_cap_peak_load_mvgd_ts_pypsa_eur_sec(mvgd_ids): export_min_cap_to_csv(df_hp_min_cap_mv_grid_pypsa_eur_sec) -def split_mvgds_into_bulks(n, max_n, func): +def split_mvgds_into_bulks(n, max_n, func, scenario=None): """ Generic function to split task into multiple parallel tasks, dividing the number of MVGDs into even bulks. @@ -1968,7 +2185,11 @@ def split_mvgds_into_bulks(n, max_n, func): mvgd_ids = mvgd_ids[n] logger.info(f"Bulk takes care of MVGD: {min(mvgd_ids)} : {max(mvgd_ids)}") - func(mvgd_ids) + + if scenario is not None: + func(mvgd_ids, scenario=scenario) + else: + func(mvgd_ids) def delete_hp_capacity(scenario): @@ -2011,12 +2232,26 @@ def delete_hp_capacity_100RE(): delete_hp_capacity(scenario="eGon100RE") +def delete_hp_capacity_status_quo(scenario): + """Remove all hp capacities for the selected status quo""" + EgonHpCapacityBuildings.__table__.create(bind=engine, checkfirst=True) + delete_hp_capacity(scenario=scenario) + + def delete_hp_capacity_2035(): """Remove all hp capacities for the selected eGon2035""" EgonHpCapacityBuildings.__table__.create(bind=engine, checkfirst=True) delete_hp_capacity(scenario="eGon2035") +def delete_mvgd_ts_status_quo(scenario): + """Remove all mvgd ts for the selected status quo""" + EgonEtragoTimeseriesIndividualHeating.__table__.create( + bind=engine, checkfirst=True + ) + delete_mvgd_ts(scenario=scenario) + + def delete_mvgd_ts_2035(): """Remove all mvgd ts for the selected eGon2035""" EgonEtragoTimeseriesIndividualHeating.__table__.create( @@ -2033,6 +2268,16 @@ def delete_mvgd_ts_100RE(): delete_mvgd_ts(scenario="eGon100RE") +def delete_heat_peak_loads_status_quo(scenario): + """Remove all heat peak loads for status quo.""" + BuildingHeatPeakLoads.__table__.create(bind=engine, checkfirst=True) + with db.session_scope() as session: + # Buses + session.query(BuildingHeatPeakLoads).filter( + BuildingHeatPeakLoads.scenario == scenario + ).delete(synchronize_session=False) + + def delete_heat_peak_loads_2035(): """Remove all heat peak loads for eGon2035.""" BuildingHeatPeakLoads.__table__.create(bind=engine, checkfirst=True) diff --git a/src/egon/data/datasets/hydrogen_etrago/bus.py b/src/egon/data/datasets/hydrogen_etrago/bus.py index c83bcd910..a2d101bf0 100755 --- a/src/egon/data/datasets/hydrogen_etrago/bus.py +++ b/src/egon/data/datasets/hydrogen_etrago/bus.py @@ -13,8 +13,12 @@ potential H2 saltcaverns. """ +import datetime +import json from geoalchemy2 import Geometry +from sqlalchemy import BigInteger, Column, Text +from sqlalchemy.ext.declarative import declarative_base from egon.data import config, db from egon.data.datasets.etrago_helpers import ( @@ -22,6 +26,13 @@ finalize_bus_insertion, initialise_bus_insertion, ) +from egon.data.metadata import ( + context, + contributors, + license_egon_data_odbl, + meta_metadata, + sources, +) def insert_hydrogen_buses(scenario="eGon2035"): @@ -61,6 +72,89 @@ def insert_hydrogen_buses(scenario="eGon2035"): insert_H2_buses_from_CH4_grid(hydrogen_buses, carrier, target, scenario) +Base = declarative_base() + + +class EgonMapACH2(Base): + source_list = [ + sources()["openstreetmap"], + sources()["SciGRID_gas"], + sources()["bgr_inspeeds_data_bundle"], + ] + meta_ac_h2 = { + "name": "grid.egon_etrago_ac_h2", + "title": "Mapping table of AC-H2 buses", + "id": "WILL_BE_SET_AT_PUBLICATION", + "description": "Table mapping AC and H2 buses in Germany", + "language": ["en-EN"], + "publicationDate": datetime.date.today().isoformat(), + "context": context(), + "spatial": { + "location": None, + "extent": "Germany", + "resolution": None, + }, + "sources": source_list, + "licenses": [license_egon_data_odbl()], + "contributors": contributors(["fw"]), + "resources": [ + { + "profile": "tabular-data-resource", + "name": "grid.egon_etrago_ac_h2", + "path": None, + "format": "PostgreSQL", + "encoding": "UTF-8", + "schema": { + "fields": [ + { + "name": "scn_name", + "description": "Name of the scenario", + "type": "str", + "unit": None, + }, + { + "name": "bus_H2", + "description": "H2 bus_id", + "type": "integer", + "unit": None, + }, + { + "name": "bus_AC", + "description": "AC bus_id", + "type": "integer", + "unit": None, + }, + ], + "primaryKey": ["scn_name", "bus_H2"], + "foreignKeys": [], + }, + "dialect": {"delimiter": None, "decimalSeparator": "."}, + } + ], + "metaMetadata": meta_metadata(), + } + # Create json dump + meta_json_ac_h2 = ( + "'" + json.dumps(meta_ac_h2, indent=4, ensure_ascii=False) + "'" + ) + + __tablename__ = "egon_etrago_ac_h2" + __table_args__ = { + "schema": "grid", + "comment": meta_json_ac_h2, + } + + scn_name = Column(Text, primary_key=True, nullable=False) + bus_H2 = Column(BigInteger, primary_key=True, nullable=False) + bus_AC = Column(BigInteger, primary_key=False, nullable=False) + + +def create_AC_H2_table(): + engine = db.engine() + EgonMapACH2.__table__.drop(bind=engine, checkfirst=True) + EgonMapACH2.__table__.create(bind=engine, checkfirst=True) + + def insert_H2_buses_from_saltcavern(gdf, carrier, sources, target, scn_name): """ Insert the H2 buses based on saltcavern locations into the database. @@ -123,16 +217,99 @@ def insert_H2_buses_from_saltcavern(gdf, carrier, sources, target, scn_name): gdf_H2_cavern["bus_AC"] = AC_bus_ids gdf_H2_cavern["scn_name"] = hydrogen_bus_ids["scn_name"] + create_AC_H2_table() + # Insert data to db gdf_H2_cavern.to_sql( "egon_etrago_ac_h2", db.engine(), schema="grid", index=False, - if_exists="replace", + if_exists="append", ) +class EgonMapH2CH4(Base): + source_list = [ + sources()["openstreetmap"], + sources()["SciGRID_gas"], + sources()["bgr_inspeeds_data_bundle"], + ] + meta_H2_CH4 = { + "name": "grid.egon_etrago_ch4_h2", + "title": "Mapping table of CH4-H2 buses", + "id": "WILL_BE_SET_AT_PUBLICATION", + "description": "Table mapping CH4 and H2 buses in Germany", + "language": ["en-EN"], + "publicationDate": datetime.date.today().isoformat(), + "context": context(), + "spatial": { + "location": None, + "extent": "Germany", + "resolution": None, + }, + "sources": source_list, + "licenses": [license_egon_data_odbl()], + "contributors": contributors(["fw"]), + "resources": [ + { + "profile": "tabular-data-resource", + "name": "grid.egon_etrago_ch4_h2", + "path": None, + "format": "PostgreSQL", + "encoding": "UTF-8", + "schema": { + "fields": [ + { + "name": "scn_name", + "description": "Name of the scenario", + "type": "str", + "unit": None, + }, + { + "name": "bus_H2", + "description": "H2 bus_id", + "type": "integer", + "unit": None, + }, + { + "name": "bus_CH4", + "description": "CH4 bus_id", + "type": "integer", + "unit": None, + }, + ], + "primaryKey": ["scn_name", "bus_H2"], + "foreignKeys": [], + }, + "dialect": {"delimiter": None, "decimalSeparator": "."}, + } + ], + "metaMetadata": meta_metadata(), + } + + # Create json dump + meta_json_H2_CH4 = ( + "'" + json.dumps(meta_H2_CH4, indent=4, ensure_ascii=False) + "'" + ) + + __tablename__ = "egon_etrago_ch4_h2" + __table_args__ = { + "schema": "grid", + "comment": meta_json_H2_CH4, + } + + scn_name = Column(Text, primary_key=True, nullable=False) + bus_H2 = Column(BigInteger, primary_key=True, nullable=False) + bus_CH4 = Column(BigInteger, primary_key=False, nullable=False) + + +def create_H2_CH4_table(): + engine = db.engine() + EgonMapH2CH4.__table__.drop(bind=engine, checkfirst=True) + EgonMapH2CH4.__table__.create(bind=engine, checkfirst=True) + + def insert_H2_buses_from_CH4_grid(gdf, carrier, target, scn_name): """ Insert the H2 buses based on CH4 grid into the database. @@ -178,13 +355,15 @@ def insert_H2_buses_from_CH4_grid(gdf, carrier, target, scn_name): gdf_H2_CH4["bus_CH4"] = CH4_bus_ids["bus_id"] gdf_H2_CH4["scn_name"] = CH4_bus_ids["scn_name"] + create_H2_CH4_table() + # Insert data to db gdf_H2_CH4.to_sql( "egon_etrago_ch4_h2", engine, schema="grid", index=False, - if_exists="replace", + if_exists="append", ) @@ -197,5 +376,7 @@ def insert_hydrogen_buses_eGon100RE(): """ copy_and_modify_buses( - "eGon2035", "eGon100RE", {"carrier": ["H2_grid", "H2_saltcavern"]}, + "eGon2035", + "eGon100RE", + {"carrier": ["H2_grid", "H2_saltcavern"]}, ) diff --git a/src/egon/data/datasets/industry/__init__.py b/src/egon/data/datasets/industry/__init__.py index bb4f0fb22..e6614aa2e 100644 --- a/src/egon/data/datasets/industry/__init__.py +++ b/src/egon/data/datasets/industry/__init__.py @@ -10,6 +10,7 @@ from sqlalchemy import ARRAY, Column, Float, Integer, String from sqlalchemy.ext.declarative import declarative_base import geopandas as gpd +import pandas as pd from egon.data import db from egon.data.datasets import Dataset @@ -199,8 +200,7 @@ def industrial_demand_distr(): f"""DELETE FROM {target_osm['schema']}.{target_osm['table']}""" ) - for scn in sources["demandregio"]["scenarios"]: - + for scn in egon.data.config.settings()["egon-data"]["--scenarios"]: # Select spatial information from local database # Select administrative districts (Landkreise) including its boundaries boundaries = db.select_geodataframe( @@ -251,7 +251,7 @@ def industrial_demand_distr(): landuse = landuse.rename({"index_right": "nuts3"}, axis=1) landuse_nuts3 = landuse[["area_ha", "nuts3"]] - landuse_nuts3 = landuse.groupby(["nuts3"]).sum().reset_index() + landuse_nuts3 = landuse_nuts3.groupby(["nuts3"]).sum().reset_index() # Select data on industrial sites sites = db.select_dataframe( @@ -351,8 +351,8 @@ def industrial_demand_distr(): # Create df containing all demand per wz which will be allocated to # osm areas - demand_nuts3_osm_wz = demand_nuts3_a.append( - demand_nuts3_b_osm, ignore_index=True + demand_nuts3_osm_wz = pd.concat( + [demand_nuts3_a, demand_nuts3_b_osm], ignore_index=True ) demand_nuts3_osm_wz = ( demand_nuts3_osm_wz.groupby(["nuts3", "wz"]).sum().reset_index() @@ -385,7 +385,12 @@ def industrial_demand_distr(): landuse = landuse.rename({"id": "osm_id"}, axis=1) # Remove duplicates and adjust index - landuse = landuse.groupby(["osm_id", "wz"]).sum().reset_index() + landuse = ( + landuse.drop("geom", axis="columns") + .groupby(["osm_id", "wz"]) + .sum() + .reset_index() + ) landuse.index.rename("id", inplace=True) landuse["scenario"] = scn diff --git a/src/egon/data/datasets/industry/temporal.py b/src/egon/data/datasets/industry/temporal.py index 1a2a96450..523d2a1d3 100644 --- a/src/egon/data/datasets/industry/temporal.py +++ b/src/egon/data/datasets/industry/temporal.py @@ -16,7 +16,6 @@ def identify_voltage_level(df): - """Identify the voltage_level of a grid component based on its peak load and defined thresholds. @@ -112,7 +111,7 @@ def identify_bus(load_curves, demand_area): ).drop_duplicates(subset=["id"]) # Bring both dataframes together - peak_bus = peak_hv_c.append(peak_hv_p, ignore_index=True) + peak_bus = pd.concat([peak_hv_c, peak_hv_p], ignore_index=True) # Select ehv voronoi ehv_voronoi = db.select_geodataframe( @@ -133,7 +132,7 @@ def identify_bus(load_curves, demand_area): peak_ehv = gpd.sjoin(peak_ehv, ehv_voronoi, how="inner", op="intersects") # Bring both dataframes together - peak_bus = peak_bus.append(peak_ehv, ignore_index=True) + peak_bus = pd.concat([peak_bus, peak_ehv], ignore_index=True) # Combine dataframes to bring loadcurves and bus id together curves_da = pd.merge( @@ -190,7 +189,7 @@ def calc_load_curves_ind_osm(scenario): ) # Calculate shares of industrial branches per osm area - osm_share_wz = demands_osm_area.groupby("osm_id").apply( + osm_share_wz = demands_osm_area.groupby(["osm_id"], as_index=False).apply( lambda grp: grp / grp.sum() ) @@ -213,13 +212,16 @@ def calc_load_curves_ind_osm(scenario): annual_demand_osm = demands_osm_area.groupby("osm_id").demand.sum() # Return electrical load curves per osm industrial landuse area - load_curves = calc_load_curve(share_wz_transpose, annual_demand_osm) + load_curves = calc_load_curve(share_wz_transpose, scenario, annual_demand_osm) curves_da = identify_bus(load_curves, demand_area) # Group all load curves per bus curves_bus = ( - curves_da.drop(["id"], axis=1).fillna(0).groupby("bus_id").sum() + curves_da.drop(["id", "geom"], axis=1) + .fillna(0) + .groupby("bus_id") + .sum() ) # Initalize pandas.DataFrame for export to database @@ -256,8 +258,7 @@ def insert_osm_ind_load(): "targets" ] - for scenario in ["eGon2021", "eGon2035", "eGon100RE"]: - + for scenario in egon.data.config.settings()["egon-data"]["--scenarios"]: # Delete existing data from database db.execute_sql( f""" @@ -372,7 +373,7 @@ def calc_load_curves_ind_sites(scenario): .demand ) - load_curves = calc_load_curve(share_transpose, demands_ind_sites["demand"]) + load_curves = calc_load_curve(share_transpose, scenario, demands_ind_sites["demand"]) curves_da = identify_bus(load_curves, demand_area) @@ -382,10 +383,10 @@ def calc_load_curves_ind_sites(scenario): # Group all load curves per bus and wz curves_bus = ( - curves_da.fillna(0) + curves_da.drop(["id", "geom"], axis=1) + .fillna(0) .groupby(["bus_id", "wz"]) .sum() - .drop(["id"], axis=1) ) # Initalize pandas.DataFrame for pf table load timeseries @@ -425,8 +426,7 @@ def insert_sites_ind_load(): "targets" ] - for scenario in ["eGon2021", "eGon2035", "eGon100RE"]: - + for scenario in egon.data.config.settings()["egon-data"]["--scenarios"]: # Delete existing data from database db.execute_sql( f""" diff --git a/src/egon/data/datasets/loadarea/__init__.py b/src/egon/data/datasets/loadarea/__init__.py index c5512f174..0c970aa93 100644 --- a/src/egon/data/datasets/loadarea/__init__.py +++ b/src/egon/data/datasets/loadarea/__init__.py @@ -63,17 +63,7 @@ def __init__(self, dependencies): name=self.name, version=self.version, dependencies=dependencies, - tasks=( - create_landuse_table, - PostgresOperator( - task_id="osm_landuse_extraction", - sql=resources.read_text( - __name__, "osm_landuse_extraction.sql" - ), - postgres_conn_id="egon_data", - autocommit=True, - ), - ), + tasks=(create_landuse_table, extract_osm_landuse), ) @@ -129,6 +119,12 @@ def __init__(self, dependencies): ) +def extract_osm_landuse(): + db.execute_sql_script( + os.path.dirname(__file__) + "/osm_landuse_extraction.sql" + ) + + def create_landuse_table(): """Create tables for landuse data Returns diff --git a/src/egon/data/datasets/low_flex_scenario/__init__.py b/src/egon/data/datasets/low_flex_scenario/__init__.py index a7230696a..25be3ad54 100644 --- a/src/egon/data/datasets/low_flex_scenario/__init__.py +++ b/src/egon/data/datasets/low_flex_scenario/__init__.py @@ -3,7 +3,7 @@ """ from airflow.operators.postgres_operator import PostgresOperator from sqlalchemy.ext.declarative import declarative_base -import importlib_resources as resources +from importlib_resources import files from egon.data.datasets import Dataset @@ -21,9 +21,9 @@ def __init__(self, dependencies): { PostgresOperator( task_id="low_flex_eGon2035", - sql=resources.read_text( - __name__, "low_flex_eGon2035.sql" - ), + sql=files(__name__) + .joinpath("low_flex_eGon2035.sql") + .read_text(encoding="utf-8"), postgres_conn_id="egon_data", autocommit=True, ), diff --git a/src/egon/data/datasets/mastr.py b/src/egon/data/datasets/mastr.py index eef2bfe51..d1ccbdf5e 100644 --- a/src/egon/data/datasets/mastr.py +++ b/src/egon/data/datasets/mastr.py @@ -11,7 +11,7 @@ import egon.data.config WORKING_DIR_MASTR_OLD = Path(".", "bnetza_mastr", "dump_2021-05-03") -WORKING_DIR_MASTR_NEW = Path(".", "bnetza_mastr", "dump_2022-11-17") +WORKING_DIR_MASTR_NEW = Path(".", "bnetza_mastr", "dump_2024-01-08") def download_mastr_data(): diff --git a/src/egon/data/datasets/osm/__init__.py b/src/egon/data/datasets/osm/__init__.py index 635e00e5e..a63227f7c 100644 --- a/src/egon/data/datasets/osm/__init__.py +++ b/src/egon/data/datasets/osm/__init__.py @@ -20,7 +20,7 @@ import importlib_resources as resources -from egon.data import db +from egon.data import db, logger from egon.data.config import settings from egon.data.datasets import Dataset from egon.data.metadata import ( @@ -77,8 +77,10 @@ def to_postgres(cache_size=4096): if settings()["egon-data"]["--dataset-boundary"] == "Everything": input_filename = osm_config["target"]["file"] + logger.info("Using Everything DE dataset.") else: input_filename = osm_config["target"]["file_testmode"] + logger.info("Using testmode SH dataset.") input_file = Path(".") / "openstreetmap" / input_filename style_file = ( diff --git a/src/egon/data/datasets/osmtgmod/__init__.py b/src/egon/data/datasets/osmtgmod/__init__.py index eed93bc76..4cbe64b13 100644 --- a/src/egon/data/datasets/osmtgmod/__init__.py +++ b/src/egon/data/datasets/osmtgmod/__init__.py @@ -6,11 +6,12 @@ import logging import os import shutil +import subprocess import sys import psycopg2 -from egon.data import db +from egon.data import db, logger from egon.data.config import settings from egon.data.datasets import Dataset from egon.data.datasets.osmtgmod.substation import extract @@ -20,7 +21,7 @@ def run(): - + sys.setrecursionlimit(5000) # execute osmTGmod data_config = egon.data.config.datasets() @@ -51,18 +52,47 @@ def import_osm_data(): # Delete repository if it already exists if osmtgmod_repos.exists() and osmtgmod_repos.is_dir(): - shutil.rmtree(osmtgmod_repos) + try: + status = subprocess.check_output( + ["git", "status"], cwd=(osmtgmod_repos).absolute() + ) + if status.startswith( + b"Auf Branch features/egon" + ) or status.startswith(b"On branch features/egon"): + logger.info("OsmTGmod cloned and right branch checked out.") - subproc.run( - [ - "git", - "clone", - "--single-branch", - "--branch", - "features/egon", - "https://github.com/openego/osmTGmod.git", - ] - ) + else: + subproc.run( + [ + "git", + "checkout", + "features/egon", + ] + ) + except subprocess.CalledProcessError: + shutil.rmtree(osmtgmod_repos) + subproc.run( + [ + "git", + "clone", + "--single-branch", + "--branch", + "features/egon", + "https://github.com/openego/osmTGmod.git", + ] + ) + else: + + subproc.run( + [ + "git", + "clone", + "--single-branch", + "--branch", + "features/egon", + "https://github.com/openego/osmTGmod.git", + ] + ) data_config = egon.data.config.datasets() osm_config = data_config["openstreetmap"]["original_data"] @@ -217,21 +247,21 @@ def osmtgmod( # ============================================================== # Setup logging # ============================================================== - log = logging.getLogger() - log.setLevel(logging.INFO) - logformat = logging.Formatter( - "%(asctime)s %(message)s", "%m/%d/%Y %H:%M:%S" - ) - sh = logging.StreamHandler() - sh.setFormatter(logformat) - log.addHandler(sh) - logging.info("\n\n======================\nego_otg\n======================") - logging.info("Logging to standard output...") - # catch up some log messages from evaluation of command line arguments - logging.info("Database: {}".format(config_database)) - logging.info( - "Path for configuration file and results: {}".format(config_basepath) - ) + # log = logging.getLogger() + # log.setLevel(logging.INFO) + # logformat = logging.Formatter( + # "%(asctime)s %(message)s", "%m/%d/%Y %H:%M:%S" + # ) + # sh = logging.StreamHandler() + # sh.setFormatter(logformat) + # log.addHandler(sh) + # logging.info("\n\n======================\nego_otg\n======================") + # logging.info("Logging to standard output...") + # # catch up some log messages from evaluation of command line arguments + # logging.info("Database: {}".format(config_database)) + # logging.info( + # "Path for configuration file and results: {}".format(config_basepath) + # ) # ============================================================== # read configuration from file and create folder structure # ============================================================== @@ -260,16 +290,16 @@ def osmtgmod( if not os.path.exists(result_dir): os.makedirs(result_dir) # start logging to file - logfile = os.path.join(config_basepath, config_database + ".log") - fh = logging.FileHandler(logfile) - fh.setFormatter(logformat) - log.addHandler(fh) - logging.info("Logging to file '{0}' is set up".format(logfile)) - logging.info( - "Now logging both to standard output and to file '{0}'...".format( - logfile - ) - ) + # logfile = os.path.join(config_basepath, config_database + ".log") + # fh = logging.FileHandler(logfile) + # fh.setFormatter(logformat) + # log.addHandler(fh) + # logging.info("Logging to file '{0}' is set up".format(logfile)) + # logging.info( + # "Now logging both to standard output and to file '{0}'...".format( + # logfile + # ) + # ) logging.info("\n\n======================\nego_otg\n======================") # copy config file logging.info( @@ -411,9 +441,11 @@ def osmtgmod( # beware: comments in C-like style (such as /* comment */) arn't parsed! sqlfile_without_comments = "".join( [ - line.lstrip().split("--")[0] + "\n" - if not line.lstrip().split("--")[0] == "" - else "" + ( + line.lstrip().split("--")[0] + "\n" + if not line.lstrip().split("--")[0] == "" + else "" + ) for line in sqlfile.split("\n") ] ) @@ -520,8 +552,14 @@ def to_pypsa(): """ ) - for scenario_name in ["'eGon2035'", "'eGon100RE'"]: + # for scenario_name in ["'eGon2035'", "'eGon100RE'", "'status2019'"]: + scenario_list = egon.data.config.settings()["egon-data"]["--scenarios"] + scenario_list = [ + f"'{scn}'" if not scn[1] == "'" else scn for scn in scenario_list + ] + for scenario_name in scenario_list: + # TODO maybe not needed anymore? capital_cost = get_sector_parameters( "electricity", scenario_name.replace("'", "") )["capital_cost"] @@ -627,19 +665,19 @@ def to_pypsa(): WHERE a.line_id = result.line_id AND scn_name = {scenario_name}; - -- set capital costs for eHV-lines + -- set capital costs for eHV-lines UPDATE grid.egon_etrago_line SET capital_cost = {capital_cost['ac_ehv_overhead_line']} * length WHERE v_nom > 110 AND scn_name = {scenario_name}; - -- set capital costs for HV-lines + -- set capital costs for HV-lines UPDATE grid.egon_etrago_line SET capital_cost = {capital_cost['ac_hv_overhead_line']} * length WHERE v_nom = 110 AND scn_name = {scenario_name}; - - -- set capital costs for transformers + + -- set capital costs for transformers UPDATE grid.egon_etrago_transformer a SET capital_cost = {capital_cost['transformer_380_220']} WHERE (a.bus0 IN ( @@ -687,20 +725,20 @@ def to_pypsa(): SELECT bus_id FROM grid.egon_etrago_bus WHERE v_nom = 220)) AND scn_name = {scenario_name}; - - -- set lifetime for eHV-lines + + -- set lifetime for eHV-lines UPDATE grid.egon_etrago_line - SET lifetime = {lifetime['ac_ehv_overhead_line']} + SET lifetime = {lifetime['ac_ehv_overhead_line']} WHERE v_nom > 110 AND scn_name = {scenario_name}; - -- set capital costs for HV-lines + -- set capital costs for HV-lines UPDATE grid.egon_etrago_line SET lifetime = {lifetime['ac_hv_overhead_line']} WHERE v_nom = 110 AND scn_name = {scenario_name}; - - -- set capital costs for transformers + + -- set capital costs for transformers UPDATE grid.egon_etrago_transformer a SET lifetime = {lifetime['transformer_380_220']} WHERE (a.bus0 IN ( @@ -748,7 +786,7 @@ def to_pypsa(): SELECT bus_id FROM grid.egon_etrago_bus WHERE v_nom = 220)) AND scn_name = {scenario_name}; - + -- delete buses without connection to AC grid and generation or -- load assigned @@ -771,11 +809,30 @@ def to_pypsa(): ) +def fix_transformer_snom(): + db.execute_sql( + """ + UPDATE grid.egon_etrago_transformer AS t + SET s_nom = CAST( + LEAST( + (SELECT SUM(COALESCE(l.s_nom,0)) + FROM grid.egon_etrago_line AS l + WHERE (l.bus0 = t.bus0 OR l.bus1 = t.bus0) + AND l.scn_name = t.scn_name), + (SELECT SUM(COALESCE(l.s_nom,0)) + FROM grid.egon_etrago_line AS l + WHERE (l.bus0 = t.bus1 OR l.bus1 = t.bus1) + AND l.scn_name = t.scn_name) + ) AS smallint + ); + """) + + class Osmtgmod(Dataset): def __init__(self, dependencies): super().__init__( name="Osmtgmod", - version="0.0.5", + version="0.0.7", dependencies=dependencies, tasks=( import_osm_data, @@ -784,5 +841,6 @@ def __init__(self, dependencies): extract, to_pypsa, }, + fix_transformer_snom, ), ) diff --git a/src/egon/data/datasets/power_etrago/__init__.py b/src/egon/data/datasets/power_etrago/__init__.py index e43058619..1d7d53e78 100755 --- a/src/egon/data/datasets/power_etrago/__init__.py +++ b/src/egon/data/datasets/power_etrago/__init__.py @@ -21,7 +21,7 @@ class OpenCycleGasTurbineEtrago(Dataset): def __init__(self, dependencies): super().__init__( name="OpenCycleGasTurbineEtrago", - version="0.0.1", + version="0.0.2", dependencies=dependencies, tasks=(insert_open_cycle_gas_turbines,), ) diff --git a/src/egon/data/datasets/power_etrago/match_ocgt.py b/src/egon/data/datasets/power_etrago/match_ocgt.py index bef670bbc..c67e08589 100755 --- a/src/egon/data/datasets/power_etrago/match_ocgt.py +++ b/src/egon/data/datasets/power_etrago/match_ocgt.py @@ -7,12 +7,17 @@ import numpy as np import pandas as pd -from egon.data import db +from egon.data import config, db from egon.data.datasets.etrago_setup import link_geom_from_buses from egon.data.datasets.scenario_parameters import get_sector_parameters -def insert_open_cycle_gas_turbines(scn_name="eGon2035"): +def insert_open_cycle_gas_turbines(): + for scenario in config.settings()["egon-data"]["--scenarios"]: + insert_open_cycle_gas_turbines_per_scenario(scenario) + + +def insert_open_cycle_gas_turbines_per_scenario(scn_name): """Insert gas turbine links in egon_etrago_link table. Parameters diff --git a/src/egon/data/datasets/power_plants/__init__.py b/src/egon/data/datasets/power_plants/__init__.py index 39f68ad40..2fdf5aa1f 100755 --- a/src/egon/data/datasets/power_plants/__init__.py +++ b/src/egon/data/datasets/power_plants/__init__.py @@ -1,6 +1,10 @@ """The central module containing all code dealing with power plant data. """ + +from pathlib import Path + from geoalchemy2 import Geometry +from shapely.geometry import Point from sqlalchemy import BigInteger, Column, Float, Integer, Sequence, String from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.ext.declarative import declarative_base @@ -9,8 +13,8 @@ import numpy as np import pandas as pd -from egon.data import db -from egon.data.datasets import Dataset +from egon.data import db, logger +from egon.data.datasets import Dataset, wrapped_partial from egon.data.datasets.mastr import ( WORKING_DIR_MASTR_NEW, WORKING_DIR_MASTR_OLD, @@ -33,6 +37,7 @@ ) import egon.data.config import egon.data.datasets.power_plants.assign_weather_data as assign_weather_data # noqa: E501 +import egon.data.datasets.power_plants.metadata as pp_metadata import egon.data.datasets.power_plants.pv_ground_mounted as pv_ground_mounted import egon.data.datasets.power_plants.wind_farms as wind_onshore import egon.data.datasets.power_plants.wind_offshore as wind_offshore @@ -94,7 +99,7 @@ class PowerPlants(Dataset): #: name: str = "PowerPlants" #: - version: str = "0.0.18" + version: str = "0.0.19" def __init__(self, dependencies): super().__init__( @@ -117,6 +122,7 @@ def __init__(self, dependencies): }, wind_offshore.insert, assign_weather_data.weatherId_and_busId, + pp_metadata.metadata, ), ) @@ -560,7 +566,7 @@ def assign_voltage_level_by_capacity(mastr_loc): return mastr_loc.voltage_level -def assign_bus_id(power_plants, cfg): +def assign_bus_id(power_plants, cfg, drop_missing=False): """Assigns bus_ids to power plants according to location and voltage level Parameters @@ -618,6 +624,9 @@ def assign_bus_id(power_plants, cfg): ehv_grid_districts, ).bus_id + if drop_missing: + power_plants = power_plants[~power_plants.bus_id.isnull()] + # Assert that all power plants have a bus_id assert power_plants.bus_id.notnull().all(), f"""Some power plants are not attached to a bus: {power_plants[power_plants.bus_id.isnull()]}""" @@ -885,7 +894,7 @@ def allocate_other_power_plants(): "and Technologie == 'ORCOrganicRankineCycleAnlage'" ) - mastr_sg = mastr_sludge.append(mastr_geothermal) + mastr_sg = pd.concat([mastr_sludge, mastr_geothermal]) # Insert geometry column mastr_sg = mastr_sg[~(mastr_sg["Laengengrad"].isnull())] @@ -919,7 +928,7 @@ def allocate_other_power_plants(): mastr_sg.loc[:, "Nettonennleistung"] *= 1e-3 # Merge different sources to one df - mastr_others = mastr_sg.append(mastr_combustion).reset_index() + mastr_others = pd.concat([mastr_sg, mastr_combustion]).reset_index() # Delete entries outside Schleswig-Holstein for test mode if boundary == "Schleswig-Holstein": @@ -964,3 +973,406 @@ def allocate_other_power_plants(): ) session.add(entry) session.commit() + + +def get_conventional_power_plants_non_chp(scn_name): + + cfg = egon.data.config.datasets()["power_plants"] + # Write conventional power plants in supply.egon_power_plants + common_columns = [ + "EinheitMastrNummer", + "Energietraeger", + "Nettonennleistung", + "Laengengrad", + "Breitengrad", + "Gemeinde", + "Inbetriebnahmedatum", + "EinheitBetriebsstatus", + "DatumEndgueltigeStilllegung", + ] + # import nuclear power plants + nuclear = pd.read_csv( + WORKING_DIR_MASTR_OLD / cfg["sources"]["mastr_nuclear"], + usecols=common_columns, + ) + # import combustion power plants + comb = pd.read_csv( + WORKING_DIR_MASTR_OLD / cfg["sources"]["mastr_combustion"], + usecols=common_columns + ["ThermischeNutzleistung"], + ) + + conv = pd.concat([comb, nuclear]) + + conv = conv[ + conv.Energietraeger.isin( + [ + "Braunkohle", + "Mineralölprodukte", + "Steinkohle", + "Kernenergie", + "Erdgas", + ] + ) + ] + + # drop plants that are decommissioned + conv["DatumEndgueltigeStilllegung"] = pd.to_datetime( + conv["DatumEndgueltigeStilllegung"] + ) + + # keep plants that were decommissioned after the max date + conv.loc[ + ( + conv.DatumEndgueltigeStilllegung + > egon.data.config.datasets()["mastr_new"][f"{scn_name}_date_max"] + ), + "EinheitBetriebsstatus", + ] = "InBetrieb" + + conv = conv.loc[conv.EinheitBetriebsstatus == "InBetrieb"] + + conv = conv.drop( + columns=["EinheitBetriebsstatus", "DatumEndgueltigeStilllegung"] + ) + + # convert from KW to MW + conv["Nettonennleistung"] = conv["Nettonennleistung"] / 1000 + + # drop generators installed after 2019 + conv["Inbetriebnahmedatum"] = pd.to_datetime(conv["Inbetriebnahmedatum"]) + conv = conv[ + conv["Inbetriebnahmedatum"] + < egon.data.config.datasets()["mastr_new"][f"{scn_name}_date_max"] + ] + + conv_cap_chp = ( + conv.groupby("Energietraeger")["Nettonennleistung"].sum() / 1e3 + ) + # drop chp generators + conv["ThermischeNutzleistung"] = conv["ThermischeNutzleistung"].fillna(0) + conv = conv[conv.ThermischeNutzleistung == 0] + conv_cap_no_chp = ( + conv.groupby("Energietraeger")["Nettonennleistung"].sum() / 1e3 + ) + + logger.info("Dropped CHP generators in GW") + logger.info(conv_cap_chp - conv_cap_no_chp) + + # rename carriers + # rename carriers + conv["Energietraeger"] = conv["Energietraeger"].replace( + to_replace={ + "Braunkohle": "lignite", + "Steinkohle": "coal", + "Erdgas": "gas", + "Mineralölprodukte": "oil", + "Kernenergie": "nuclear", + } + ) + + # rename columns + conv.rename( + columns={ + "EinheitMastrNummer": "gens_id", + "Energietraeger": "carrier", + "Nettonennleistung": "capacity", + "Gemeinde": "location", + }, + inplace=True, + ) + conv["bus_id"] = np.nan + conv["geom"] = gpd.points_from_xy( + conv.Laengengrad, conv.Breitengrad, crs=4326 + ) + conv.loc[(conv.Laengengrad.isna() | conv.Breitengrad.isna()), "geom"] = ( + Point() + ) + conv = gpd.GeoDataFrame(conv, geometry="geom") + + # assign voltage level by capacity + conv["voltage_level"] = np.nan + conv["voltage_level"] = assign_voltage_level_by_capacity( + conv.rename(columns={"capacity": "Nettonennleistung"}) + ) + # Add further information + conv["sources"] = [{"el_capacity": "MaStR"}] * conv.shape[0] + conv["source_id"] = conv["gens_id"].apply(lambda x: {"MastrNummer": x}) + conv["scenario"] = scn_name + + return conv + + +def power_plants_status_quo(scn_name="status2019"): + def fill_missing_bus_and_geom(gens, carrier): + # drop generators without data to get geometry. + drop_id = gens[ + (gens.geom.is_empty) + & ~(gens.location.isin(geom_municipalities.index)) + ].index + new_geom = gens["capacity"][ + (gens.geom.is_empty) + & (gens.location.isin(geom_municipalities.index)) + ] + logger.info( + f"""{len(drop_id)} {carrier} generator(s) ({int(gens.loc[drop_id, 'capacity'] + .sum())}MW) were drop""" + ) + + logger.info( + f"""{len(new_geom)} {carrier} generator(s) ({int(new_geom + .sum())}MW) received a geom based on location + """ + ) + gens.drop(index=drop_id, inplace=True) + + # assign missing geometries based on location and buses based on geom + + gens["geom"] = gens.apply( + lambda x: ( + geom_municipalities.at[x["location"], "geom"] + if x["geom"].is_empty + else x["geom"] + ), + axis=1, + ) + gens["bus_id"] = gens.sjoin( + mv_grid_districts[["bus_id", "geom"]], how="left" + ).bus_id_right.values + + gens = gens.dropna(subset=["bus_id"]) + # convert geom to WKB + gens["geom"] = gens["geom"].to_wkt() + + return gens + + def convert_master_info(df): + # Add further information + df["sources"] = [{"el_capacity": "MaStR"}] * df.shape[0] + df["source_id"] = df["gens_id"].apply(lambda x: {"MastrNummer": x}) + return df + + def log_insert_capacity(df, tech): + logger.info( + f""" + {len(df)} {tech} generators with a total installed capacity of + {int(df["el_capacity"].sum())} MW were inserted into the db + """ + ) + + con = db.engine() + cfg = egon.data.config.datasets()["power_plants"] + + db.execute_sql( + f""" + DELETE FROM {cfg['target']['schema']}.{cfg['target']['table']} + WHERE carrier IN ('wind_onshore', 'solar', 'biomass', + 'run_of_river', 'reservoir', 'solar_rooftop', + 'wind_offshore', 'nuclear', 'coal', 'lignite', 'oil', + 'gas') + AND scenario = '{scn_name}' + """ + ) + + # import municipalities to assign missing geom and bus_id + geom_municipalities = gpd.GeoDataFrame.from_postgis( + """ + SELECT gen, ST_UNION(geometry) as geom + FROM boundaries.vg250_gem + GROUP BY gen + """, + con, + geom_col="geom", + ).set_index("gen") + geom_municipalities["geom"] = geom_municipalities["geom"].centroid + + mv_grid_districts = gpd.GeoDataFrame.from_postgis( + f""" + SELECT * FROM {cfg['sources']['egon_mv_grid_district']} + """, + con, + ) + mv_grid_districts.geom = mv_grid_districts.geom.to_crs(4326) + + # Conventional non CHP + # ################### + conv = get_conventional_power_plants_non_chp(scn_name) + conv = fill_missing_bus_and_geom(conv, carrier="conventional") + conv= conv.rename(columns={"capacity": "el_capacity"}) + + # Write into DB + with db.session_scope() as session: + session.bulk_insert_mappings( + EgonPowerPlants, + conv.to_dict(orient="records"), + ) + + log_insert_capacity(conv, tech="conventional non chp") + + # Hydro Power Plants + # ################### + hydro = gpd.GeoDataFrame.from_postgis( + f"""SELECT *, city AS location FROM {cfg['sources']['hydro']} + WHERE plant_type IN ('Laufwasseranlage', 'Speicherwasseranlage')""", + con, + geom_col="geom", + ) + + hydro = fill_missing_bus_and_geom(hydro, carrier="hydro") + + hydro = convert_master_info(hydro) + hydro["carrier"] = hydro["plant_type"].replace( + to_replace={ + "Laufwasseranlage": "run_of_river", + "Speicherwasseranlage": "reservoir", + } + ) + hydro["scenario"] = scn_name + hydro = hydro.rename(columns={"capacity": "el_capacity"}) + hydro = hydro.drop(columns="id") + + # Write into DB + with db.session_scope() as session: + session.bulk_insert_mappings( + EgonPowerPlants, + hydro.to_dict(orient="records"), + ) + + log_insert_capacity(hydro, tech="hydro") + + # Biomass + # ################### + biomass = gpd.GeoDataFrame.from_postgis( + f"""SELECT *, city AS location FROM {cfg['sources']['biomass']}""", + con, + geom_col="geom", + ) + + # drop chp generators + biomass["th_capacity"] = biomass["th_capacity"].fillna(0) + biomass = biomass[biomass.th_capacity == 0] + + biomass = fill_missing_bus_and_geom(biomass, carrier="biomass") + + biomass = convert_master_info(biomass) + biomass["scenario"] = scn_name + biomass["carrier"] = "biomass" + biomass = biomass.rename(columns={"capacity": "el_capacity"}) + biomass = biomass.drop(columns="id") + + # Write into DB + with db.session_scope() as session: + session.bulk_insert_mappings( + EgonPowerPlants, + biomass.to_dict(orient="records"), + ) + + log_insert_capacity(biomass, tech="biomass") + + # Solar + # ################### + solar = gpd.GeoDataFrame.from_postgis( + f"""SELECT *, city AS location FROM {cfg['sources']['pv']} + WHERE site_type IN ('Freifläche', + 'Bauliche Anlagen (Hausdach, Gebäude und Fassade)') """, + con, + geom_col="geom", + ) + map_solar = { + "Freifläche": "solar", + "Bauliche Anlagen (Hausdach, Gebäude und Fassade)": "solar_rooftop", + } + solar["carrier"] = solar["site_type"].replace(to_replace=map_solar) + + solar = fill_missing_bus_and_geom(solar, carrier="solar") + solar = convert_master_info(solar) + solar["scenario"] = scn_name + solar = solar.rename(columns={"capacity": "el_capacity"}) + solar = solar.drop(columns="id") + + # Write into DB + with db.session_scope() as session: + session.bulk_insert_mappings( + EgonPowerPlants, + solar.to_dict(orient="records"), + ) + + log_insert_capacity(solar, tech="solar") + + # Wind + # ################### + wind_onshore = gpd.GeoDataFrame.from_postgis( + f"""SELECT *, city AS location FROM {cfg['sources']['wind']}""", + con, + geom_col="geom", + ) + + wind_onshore = fill_missing_bus_and_geom( + wind_onshore, carrier="wind_onshore" + ) + wind_onshore = convert_master_info(wind_onshore) + wind_onshore["scenario"] = scn_name + wind_onshore = wind_onshore.rename(columns={"capacity": "el_capacity"}) + wind_onshore["carrier"] = "wind_onshore" + wind_onshore = wind_onshore.drop(columns="id") + + # Write into DB + with db.session_scope() as session: + session.bulk_insert_mappings( + EgonPowerPlants, + wind_onshore.to_dict(orient="records"), + ) + + log_insert_capacity(wind_onshore, tech="wind_onshore") + + + +tasks = ( + create_tables, + import_mastr, +) + +for scn_name in egon.data.config.settings()["egon-data"]["--scenarios"]: + if "status" in scn_name: + tasks += ( + wrapped_partial( + power_plants_status_quo, + scn_name=scn_name, + postfix=f"_{scn_name[-4:]}", + ), + ) + +if ( + "eGon2035" in egon.data.config.settings()["egon-data"]["--scenarios"] + or "eGon100RE" in egon.data.config.settings()["egon-data"]["--scenarios"] +): + tasks = tasks + ( + insert_hydro_biomass, + allocate_conventional_non_chp_power_plants, + allocate_other_power_plants, + { + wind_onshore.insert, + pv_ground_mounted.insert, + pv_rooftop_per_mv_grid, + }, + ) + +tasks = tasks + ( + pv_rooftop_to_buildings, + wind_offshore.insert, +) + +for scn_name in egon.data.config.settings()["egon-data"]["--scenarios"]: + tasks += (wrapped_partial(assign_weather_data.weatherId_and_busId, + scn_name=scn_name, + # postfix=f"_{scn_name}" + ),) + + +class PowerPlants(Dataset): + def __init__(self, dependencies): + super().__init__( + name="PowerPlants", + version="0.0.26", + dependencies=dependencies, + tasks=tasks, + ) diff --git a/src/egon/data/datasets/power_plants/assign_weather_data.py b/src/egon/data/datasets/power_plants/assign_weather_data.py index e650699a9..38e3f80ae 100644 --- a/src/egon/data/datasets/power_plants/assign_weather_data.py +++ b/src/egon/data/datasets/power_plants/assign_weather_data.py @@ -1,155 +1,141 @@ import geopandas as gpd -import pandas as pd from egon.data import db -import egon.data.config -import egon.data.datasets.power_plants.__init__ as init_pp +from egon.data.datasets.power_plants.pv_rooftop_buildings import timer_func -def weatherId_and_busId(): - power_plants, cfg, con = find_weather_id() - power_plants = find_bus_id(power_plants, cfg) - write_power_plants_table(power_plants, cfg, con) - - -def find_bus_id(power_plants, cfg): - # Define bus_id for power plants without it - power_plants_no_busId = power_plants[power_plants.bus_id.isna()] - power_plants = power_plants[~power_plants.bus_id.isna()] - - power_plants_no_busId = power_plants_no_busId.drop(columns="bus_id") - - if len(power_plants_no_busId) > 0: - power_plants_no_busId = init_pp.assign_bus_id( - power_plants_no_busId, cfg - ) - - power_plants = power_plants.append(power_plants_no_busId) - - return power_plants - - -def find_weather_id(): - """ - Assign weather data to the weather dependant generators (wind and solar) +def assign_bus_id(power_plants, cfg): + """Assigns bus_ids to power plants according to location and voltage level Parameters ---------- - *No parameters required - """ - - # Connect to the data base - con = db.engine() + power_plants : pandas.DataFrame + Power plants including voltage level - cfg = egon.data.config.datasets()["weather_BusID"] + Returns + ------- + power_plants : pandas.DataFrame + Power plants including voltage level and bus_id - # Import table with power plants - sql = f""" - SELECT * FROM - {cfg['sources']['power_plants']['schema']}. - {cfg['sources']['power_plants']['table']} - """ - power_plants = gpd.GeoDataFrame.from_postgis( - sql, con, crs="EPSG:4326", geom_col="geom" - ) - - # select the power_plants that are weather dependant (wind offshore is - # not included here, because it alredy has weather_id assigned) - power_plants = power_plants[ - (power_plants["carrier"] == "solar") - | (power_plants["carrier"] == "wind_onshore") - ] - power_plants.set_index("id", inplace=True) - - # Import table with weather data for each technology - sql = f""" - SELECT * FROM - {cfg['sources']['renewable_feedin']['schema']}. - {cfg['sources']['renewable_feedin']['table']} """ - weather_data = pd.read_sql_query(sql, con) - weather_data.set_index("w_id", inplace=True) - # Import weather cells with Id to match with the weather data - sql = f""" - SELECT * FROM - {cfg['sources']['weather_cells']['schema']}. - {cfg['sources']['weather_cells']['table']} - """ - weather_cells = gpd.GeoDataFrame.from_postgis( - sql, con, crs="EPSG:4326", geom_col="geom" + mv_grid_districts = db.select_geodataframe( + f""" + SELECT * FROM {cfg['sources']['egon_mv_grid_district']} + """, + epsg=4326, ) - # import Germany borders to speed up the matching process - sql = "SELECT * FROM boundaries.vg250_sta" - sql = f""" - SELECT * FROM - {cfg['sources']['boundaries']['schema']}. - {cfg['sources']['boundaries']['table']} - """ - boundaries = gpd.GeoDataFrame.from_postgis( - sql, con, crs="EPSG:4326", geom_col="geometry" + ehv_grid_districts = db.select_geodataframe( + f""" + SELECT * FROM {cfg['sources']['ehv_voronoi']} + """, + epsg=4326, ) - # Clip weater data cells using the German boundaries - weather_cells = gpd.clip(weather_cells, boundaries) - - for weather_id in weather_cells["w_id"]: - df = gpd.clip( - power_plants, weather_cells[weather_cells["w_id"] == weather_id] + # Assign power plants in hv and below to hvmv bus + power_plants_hv = power_plants[power_plants.voltage_level >= 3].index + if len(power_plants_hv) > 0: + power_plants.loc[power_plants_hv, "bus_id"] = gpd.sjoin( + power_plants[power_plants.index.isin(power_plants_hv)], + mv_grid_districts, + ).bus_id + + # Assign power plants in ehv to ehv bus + power_plants_ehv = power_plants[power_plants.voltage_level < 3].index + + if len(power_plants_ehv) > 0: + ehv_join = gpd.sjoin( + power_plants[power_plants.index.isin(power_plants_ehv)], + ehv_grid_districts, ) - power_plant_list = df.index.to_list() - power_plants.loc[power_plant_list, "weather_cell_id"] = weather_id - return (power_plants, cfg, con) + if "bus_id_right" in ehv_join.columns: + power_plants.loc[power_plants_ehv, "bus_id"] = gpd.sjoin( + power_plants[power_plants.index.isin(power_plants_ehv)], + ehv_grid_districts, + ).bus_id_right + else: + power_plants.loc[power_plants_ehv, "bus_id"] = gpd.sjoin( + power_plants[power_plants.index.isin(power_plants_ehv)], + ehv_grid_districts, + ).bus_id -def write_power_plants_table(power_plants, cfg, con): + # Assert that all power plants have a bus_id + assert power_plants.bus_id.notnull().all(), f"""Some power plants are + not attached to a bus: {power_plants[power_plants.bus_id.isnull()]}""" - # delete weather dependent power_plants from supply.egon_power_plants - db.execute_sql( - f""" - DELETE FROM {cfg['sources']['power_plants']['schema']}. - {cfg['sources']['power_plants']['table']} - WHERE carrier IN ('wind_onshore', 'solar') - """ - ) + return power_plants - # assert that the column "bus_id" is set as integer - power_plants["bus_id"] = power_plants["bus_id"].apply( - lambda x: pd.NA if pd.isna(x) else int(x) - ) - # assert that the column "weather_cell_id" is set as integer - power_plants["weather_cell_id"] = power_plants["weather_cell_id"].apply( - lambda x: pd.NA if pd.isna(x) else int(x) - ) +@timer_func +def add_missing_bus_ids(scn_name): + """Assign busses by spatal intersection of mvgrid districts or ehv voronois.""" - # Look for the maximum id in the table egon_power_plants sql = f""" - SELECT MAX(id) FROM - {cfg['sources']['power_plants']['schema']}. - {cfg['sources']['power_plants']['table']} - """ - max_id = pd.read_sql(sql, con) - max_id = max_id["max"].iat[0] - if max_id is None: - ini_id = 1 - else: - ini_id = int(max_id + 1) - - # write_table in egon-data database: - # Reset index - power_plants.index = pd.RangeIndex( - start=ini_id, stop=ini_id + len(power_plants), name="id" - ) - - # Insert into database - power_plants.reset_index().to_postgis( - name=f"{cfg['sources']['power_plants']['table']}", - schema=f"{cfg['sources']['power_plants']['schema']}", - con=con, - if_exists="append", - ) - - return "Bus_id and Weather_id were updated succesfully" + -- Assign missing buses to mv grid district buses for HV and below + UPDATE supply.egon_power_plants AS epp + SET bus_id = ( + SELECT emgd.bus_id + FROM grid.egon_mv_grid_district AS emgd + WHERE ST_Intersects(epp.geom, emgd.geom) + ORDER BY emgd.geom <-> epp.geom + LIMIT 1 + ) + WHERE (epp.carrier = 'solar' + OR epp.carrier = 'wind_onshore' + OR epp.carrier = 'solar_rooftop' + OR epp.carrier = 'wind_offshore') + AND epp.scenario = '{scn_name}' + AND epp.bus_id is null + AND epp.voltage_level >= 3; -- HV and below + + + -- Assign missing buses to EHV buses for EHV + UPDATE supply.egon_power_plants AS epp + SET bus_id = ( + SELECT eesv.bus_id + FROM grid.egon_ehv_substation_voronoi AS eesv + WHERE ST_Intersects(epp.geom, eesv.geom) + ORDER BY eesv.geom <-> epp.geom + LIMIT 1 + ) + WHERE (epp.carrier = 'solar' + OR epp.carrier = 'wind_onshore' + OR epp.carrier = 'solar_rooftop' + OR epp.carrier = 'wind_offshore') + AND epp.scenario = '{scn_name}' + AND epp.bus_id is null + AND epp.voltage_level < 3; --EHV + + + """ + + db.execute_sql(sql) + + +@timer_func +def find_weather_id(scn_name): + + sql = f"""UPDATE supply.egon_power_plants AS epp + SET weather_cell_id = ( + SELECT eewc.w_id + FROM supply.egon_era5_weather_cells AS eewc + WHERE ST_Intersects(epp.geom, eewc.geom) + ORDER BY eewc.geom <-> epp.geom + LIMIT 1 + ) + WHERE (epp.carrier = 'solar' + OR epp.carrier = 'solar_rooftop' + OR epp.carrier = 'wind_onshore' + OR epp.carrier = 'wind_offshore') + AND epp.scenario = '{scn_name}'; + """ + + db.execute_sql(sql) + + +def weatherId_and_busId(scn_name): + find_weather_id(scn_name) + add_missing_bus_ids(scn_name) diff --git a/src/egon/data/datasets/power_plants/conventional.py b/src/egon/data/datasets/power_plants/conventional.py index c0440192b..41226730f 100644 --- a/src/egon/data/datasets/power_plants/conventional.py +++ b/src/egon/data/datasets/power_plants/conventional.py @@ -144,11 +144,9 @@ def match_nep_no_chp( ) for ET in nep["carrier"].unique(): - for index, row in nep[ (nep["carrier"] == ET) & (nep["postcode"] != "None") ].iterrows(): - # Select plants from MaStR that match carrier, PLZ # and have a similar capacity # Create a copy of all power plants from MaStR @@ -191,18 +189,25 @@ def match_nep_no_chp( # If a plant could be matched, add this to matched if len(selected) > 0: - matched = matched.append( - gpd.GeoDataFrame( - data={ - "source": "MaStR scaled with NEP 2021 list", - "MaStRNummer": selected.EinheitMastrNummer.head(1), - "carrier": ET, - "el_capacity": row.c2035_capacity, - "scenario": "eGon2035", - "geometry": selected.geometry.head(1), - "voltage_level": selected.voltage_level.head(1), - } - ) + matched = pd.concat( + [ + matched, + gpd.GeoDataFrame( + data={ + "source": "MaStR scaled with NEP 2021 list", + "MaStRNummer": selected.EinheitMastrNummer.head( + 1 + ), + "carrier": ET, + "el_capacity": row.c2035_capacity, + "scenario": "eGon2035", + "geometry": selected.geometry.head(1), + "voltage_level": selected.voltage_level.head( + 1 + ), + } + ), + ] ) # Drop matched power plant from nep diff --git a/src/egon/data/datasets/power_plants/mastr.py b/src/egon/data/datasets/power_plants/mastr.py index 225c7c685..dc8867752 100644 --- a/src/egon/data/datasets/power_plants/mastr.py +++ b/src/egon/data/datasets/power_plants/mastr.py @@ -33,6 +33,7 @@ from egon.data import config, db from egon.data.datasets.mastr import WORKING_DIR_MASTR_NEW from egon.data.datasets.power_plants.mastr_db_classes import ( + add_metadata, EgonMastrGeocoded, EgonPowerPlantsBiomass, EgonPowerPlantsCombustion, @@ -349,7 +350,9 @@ def import_mastr() -> None: # drop units installed after reference date from cfg # (eGon2021 scenario) len_old = len(units) - ts = pd.Timestamp(config.datasets()["mastr_new"]["egon2021_date_max"]) + ts = pd.Timestamp( + config.datasets()["mastr_new"]["status2023_date_max"] + ) units = units.loc[pd.to_datetime(units.Inbetriebnahmedatum) <= ts] logger.debug( f"{len_old - len(units)} units installed after {ts} dropped..." @@ -528,3 +531,5 @@ def import_mastr() -> None: if_exists="append", schema=target_tables[tech].__table_args__["schema"], ) + + add_metadata() diff --git a/src/egon/data/datasets/power_plants/mastr_db_classes.py b/src/egon/data/datasets/power_plants/mastr_db_classes.py index 7891dd796..43a2af6d2 100644 --- a/src/egon/data/datasets/power_plants/mastr_db_classes.py +++ b/src/egon/data/datasets/power_plants/mastr_db_classes.py @@ -1,4 +1,8 @@ +import datetime +import json + from geoalchemy2 import Geometry +from omi.dialects import get_dialect from sqlalchemy import ( Boolean, Column, @@ -10,6 +14,17 @@ ) from sqlalchemy.ext.declarative import declarative_base +from egon.data import config, db +from egon.data.metadata import ( + context, + contributors, + generate_resource_fields_from_db_table, + license_dedl, + meta_metadata, + meta_metadata, + sources, +) + Base = declarative_base() @@ -280,3 +295,154 @@ class EgonPowerPlantsStorage(Base): geometry_geocoded = Column(Boolean) geom = Column(Geometry("POINT", 4326), index=True, nullable=True) + + +def add_metadata(): + technologies = config.datasets()["mastr_new"]["technologies"] + + target_tables = { + "pv": EgonPowerPlantsPv, + "wind": EgonPowerPlantsWind, + "biomass": EgonPowerPlantsBiomass, + "hydro": EgonPowerPlantsHydro, + "combustion": EgonPowerPlantsCombustion, + "gsgk": EgonPowerPlantsGsgk, + "nuclear": EgonPowerPlantsNuclear, + "storage": EgonPowerPlantsStorage, + } + + deposit_id_data_bundle = config.datasets()["data-bundle"]["sources"][ + "zenodo" + ]["deposit_id"] + deposit_id_mastr = config.datasets()["mastr_new"]["deposit_id"] + + contris = contributors(["kh", "kh"]) + + contris[0]["date"] = "2023-03-15" + + contris[0]["object"] = "metadata" + contris[1]["object"] = "dataset" + + contris[0]["comment"] = "Add metadata to dataset." + contris[1]["comment"] = "Add workflow to generate dataset." + + for technology in technologies: + target_table = target_tables[technology] + + meta = { + "name": ( + f"{target_table.__table_args__['schema']}." + f"{target_table.__tablename__}" + ), + "title": f"eGon {technology} power plants", + "id": "WILL_BE_SET_AT_PUBLICATION", + "description": ( + f"eGon {technology} power plants status quo derived from MaStR" + ), + "language": "en-US", + "keywords": [technology, "mastr"], + "publicationDate": datetime.date.today().isoformat(), + "context": context(), + "spatial": { + "location": "none", + "extent": "Germany", + "resolution": "1 m", + }, + "temporal": { + "referenceDate": ( + config.datasets()["mastr_new"]["egon2021_date_max"].split( + " " + )[0] + ), + "timeseries": {}, + }, + "sources": [ + { + "title": "Data bundle for egon-data", + "description": ( + "Data bundle for egon-data: A transparent and " + "reproducible data processing pipeline for energy " + "system modeling" + ), + "path": ( + "https://zenodo.org/record/" + f"{deposit_id_data_bundle}#.Y_dWM4CZMVM" + ), + "licenses": [license_dedl(attribution="© Cußmann, Ilka")], + }, + { + "title": ( + "open-MaStR power unit registry for eGo^n project" + ), + "description": ( + "Data from Marktstammdatenregister (MaStR) data using " + "the data dump from 2022-11-17 for eGon-data." + ), + "path": ( + f"https://zenodo.org/record/{deposit_id_mastr}" + ), + "licenses": [license_dedl(attribution="© Amme, Jonathan")], + }, + sources()["egon-data"], + ], + "licenses": [license_dedl(attribution="© eGon development team")], + "contributors": contris, + "resources": [ + { + "profile": "tabular-data-resource", + "name": ( + f"{target_table.__table_args__['schema']}." + f"{target_table.__tablename__}" + ), + "path": "None", + "format": "PostgreSQL", + "encoding": "UTF-8", + "schema": { + "fields": generate_resource_fields_from_db_table( + target_table.__table_args__["schema"], + target_table.__tablename__, + geom_columns=["geom"], + ), + "primaryKey": "id", + }, + "dialect": {"delimiter": "", "decimalSeparator": ""}, + } + ], + "review": {"path": "", "badge": ""}, + "metaMetadata": meta_metadata(), + "_comment": { + "metadata": ( + "Metadata documentation and explanation (https://github." + "com/OpenEnergyPlatform/oemetadata/blob/master/metadata/" + "v141/metadata_key_description.md)" + ), + "dates": ( + "Dates and time must follow the ISO8601 including time " + "zone (YYYY-MM-DD or YYYY-MM-DDThh:mm:ss±hh)" + ), + "units": "Use a space between numbers and units (100 m)", + "languages": ( + "Languages must follow the IETF (BCP47) format (en-GB, " + "en-US, de-DE)" + ), + "licenses": ( + "License name must follow the SPDX License List " + "(https://spdx.org/licenses/)" + ), + "review": ( + "Following the OEP Data Review (https://github.com/" + "OpenEnergyPlatform/data-preprocessing/wiki)" + ), + "none": "If not applicable use (none)", + }, + } + + dialect = get_dialect(f"oep-v{meta_metadata()['metadataVersion'][4:7]}")() + + meta = dialect.compile_and_render(dialect.parse(json.dumps(meta))) + + db.submit_comment( + f"'{json.dumps(meta)}'", + target_table.__table_args__["schema"], + target_table.__tablename__, + ) diff --git a/src/egon/data/datasets/power_plants/metadata.py b/src/egon/data/datasets/power_plants/metadata.py new file mode 100644 index 000000000..703fd3f33 --- /dev/null +++ b/src/egon/data/datasets/power_plants/metadata.py @@ -0,0 +1,266 @@ +"""Metadata for power plants table +""" +import datetime +import json + +from egon.data import db +from egon.data.metadata import ( + license_agpl, + license_ccby, + licenses_datenlizenz_deutschland, +) + + +def metadata(): + """Add metdata to power plants table + + Returns + ------- + None. + + """ + meta = { + "name": "supply.egon_power_plants", + "title": "supply.egon_power_plants", + "id": "", + "description": "Database of powerplants ", + "language": "en-GB", + "keywords": [], + "publicationDate": datetime.date.today().isoformat(), + "context": { + "homepage": "https://ego-n.org/tools_data/", + "documentation": "https://egon-data.readthedocs.io/en/latest/", + "sourceCode": "https://github.com/openego/eGon-data", + "contact": "https://ego-n.org/partners/", + "grantNo": "03EI1002", + "fundingAgency": "Bundesministerium für Wirtschaft und Energie", + "fundingAgencyLogo": "https://www.innovation-beratung-foerderung.de/INNO/Redaktion/DE/Bilder/Titelbilder/titel_foerderlogo_bmwi.jpg?__blob=normal&v=3", + "publisherLogo": "https://ego-n.org/images/eGon_logo_noborder_transbg.svg", + }, + "spatial": {"location": "", "extent": "Germany", "resolution": ""}, + "sources": [ + { + "title": '"open-MaStR power unit registry"', + "description": "Raw data download Marktstammdatenregister (MaStR) data using the webservice. All data from the Marktstammdatenregister is included. There are duplicates included. For further information read in the documentationg of the original data source: https://www.marktstammdatenregister.de/MaStRHilfe/subpages/statistik.html", + "path": "https://zenodo.org/record/10480958", + "licenses": [ + { + "name": licenses_datenlizenz_deutschland( + '"© 2021 Bundesnetzagentur für Elektrizität, Gas, Telekommunikation, Post und Eisenbahnen"' + ), + "title": "", + "path": "", + "instruction": "", + "attribution": "eGon developers", + } + ], + }, + { + "title": '"ERA5 global reanalysis"', + "description": "ERA5 is the fifth generation ECMWF reanalysis for the global climate " + "and weather for the past 4 to 7 decades. Currently data is available from 1950, " + "split into Climate Data Store entries for 1950-1978 (preliminary back extension) and f" + "rom 1979 onwards (final release plus timely updates, this page). ERA5 replaces the ERA-Interim reanalysis. " + "See the online ERA5 documentation " + "(https://confluence.ecmwf.int/display/CKB/ERA5%3A+data+documentation#ERA5:datadocumentation-Dataupdatefrequency) " + "for more information.", + "path": "https://confluence.ecmwf.int/display/CKB/ERA5%3A+data+documentation#ERA5:datadocumentation-Dataupdatefrequency", + "licenses": [ + { + "name": "Licence to use Copernicus Products", + "title": "Licence to use Copernicus Products", + "path": "https://cds.climate.copernicus.eu/api/v2/terms/static/licence-to-use-copernicus-products.pdf", + "instruction": "This Licence is free of charge, worldwide, non-exclusive, royalty free and perpetual. " + "Access to Copernicus Products is given for any purpose in so far as it is lawful, whereas use " + "may include, but is not limited to: reproduction; distribution; communication to the public; " + "adaptation, modification and combination with other data and information; or any " + "combination of the foregoing", + "attribution": "Copernicus Climate Change Service (C3S) Climate Data Store", + }, + ], + }, + { + "title": '"Verwaltungsgebiete 1:250 000 (Ebenen)"', + "description": "Der Datenbestand umfasst sämtliche Verwaltungseinheiten der " + "hierarchischen Verwaltungsebenen vom Staat bis zu den Gemeinden " + "mit ihren Grenzen, statistischen Schlüsselzahlen, Namen der " + "Verwaltungseinheit sowie die spezifische Bezeichnung der " + "Verwaltungsebene des jeweiligen Landes.", + "path": "https://daten.gdz.bkg.bund.de/produkte/vg/vg250_ebenen_0101/2020/vg250_01-01.geo84.shape.ebenen.zip", + "licenses": [ + licenses_datenlizenz_deutschland( + "© Bundesamt für Kartographie und Geodäsie " + "2020 (Daten verändert)" + ) + ], + }, + { + "title": '"eGon-data"', + "description": "Workflow to download, process and generate data sets" + "suitable for the further research conducted in the project eGon (https://ego-n.org/)", + "path": "https://github.com/openego/eGon-data", + "licenses": [ + license_agpl( + "© Jonathan Amme, Clara Büttner, Ilka Cußmann, Julian Endres, Carlos Epia, Stephan Günther, Ulf Müller, Amélia Nadal, Guido Pleßmann, Francesco Witte" + ) + ], + }, + { + "title": '"Netzentwicklungsplan Strom 2035, Version 2021, erster Entwurf"', + "description": "Die vier deutschen Übertragungsnetzbetreiber zeigen mit " + "diesem ersten Entwurf des Netzentwicklungsplans 2035, Version 2021, den " + "benötigten Netzausbau für die nächsten Jahre auf. Der NEP-Bericht beschreibt " + "keine konkreten Trassenverläufe von Übertragungsleitungen, sondern er " + "dokumentiert den notwendigen Übertragungsbedarf zwischen Netzknoten. " + "Das heißt, es werden Anfangs- und Endpunkte von zukünftigen Leitungsverbindungen " + "definiert sowie konkrete Empfehlungen für den Aus- und Neubau der Übertragungsnetze " + "an Land und auf See in Deutschland gemäß den Detailanforderungen im § 12 EnWG gegeben.", + "path": "https://zenodo.org/record/5743452#.YbCoz7so8go", + "licenses": [license_ccby("© Übertragungsnetzbetreiber")], + }, + { + "title": '"Data bundle for egon-data"', + "description": "Zenodo repository to provide several different input data sets for eGon-data", + "path": "https://zenodo.org/record/10226009", + "licenses": [license_ccby("© eGon development team")], + }, + ], + "licenses": [ + { + "name": "CC-BY-4.0", + "title": "Creative Commons Attribution 4.0 International", + "path": "https://creativecommons.org/licenses/by/4.0/legalcode", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute.", + "attribution": "eGon developers", + } + ], + "contributors": [ + { + "title": "Ilka Cußmann", + "email": "", + "object": "", + "comment": "Added hydro and biomass plants", + }, + { + "title": "Clara Büttner", + "email": "", + "object": "", + "comment": "Added table structure", + }, + { + "title": "Katharina Esterl", + "email": "", + "object": "", + "comment": "Add pv ground mounted", + }, + { + "title": "Carlos Epia", + "email": "", + "object": "", + "comment": "Added wind power plants", + }, + ], + "resources": [ + { + "profile": "tabular-data-resource", + "name": "supply.egon_power_plants", + "path": "", + "format": "PostgreSQL", + "encoding": "UTF-8", + "schema": { + "fields": [ + { + "name": "id", + "description": "Unique identifier for power plants", + "type": "Integer ", + "unit": "", + }, + { + "name": "sources", + "description": "sources of power plants", + "type": "Dictionary", + "unit": "", + }, + { + "name": "source_id", + "description": "Unique Identifier of sources ", + "type": "Dictionary", + "unit": "", + }, + { + "name": "carrier", + "description": "Energy carrier such as biomass, wind_onshore", + "type": "String", + "unit": "", + }, + { + "name": "el_capacity", + "description": "Installed electrical capacity in MW", + "type": "", + "unit": "MW", + }, + { + "name": "bus_id", + "description": "bus ID where the power plant is connected", + "type": "Integer", + "unit": "", + }, + { + "name": "voltage_level", + "description": "voltage level of power plant", + "type": "Integer", + "unit": "", + }, + { + "name": "weather_cell_id", + "description": "Unique identifier of the corresponding weather cell", + "type": "Integer", + "unit": "", + }, + { + "name": "scenario", + "description": "eGon scenario, for example eGon235 or eGon100RE", + "type": "String", + "unit": "", + }, + { + "name": "geometry", + "description": "geometry of the power plant", + "type": "String", + "unit": "", + }, + { + "name": "", + "description": "", + "type": "", + "unit": "", + }, + ], + "primaryKey": "", + }, + "dialect": {"delimiter": "", "decimalSeparator": ""}, + } + ], + "review": {"path": "", "badge": ""}, + "metaMetadata": { + "metadataVersion": "OEP-1.4.1", + "metadataLicense": { + "name": "CC0-1.0", + "title": "Creative Commons Zero v1.0 Universal", + "path": "https://creativecommons.org/publicdomain/zero/1.0/", + }, + }, + "_comment": { + "metadata": "Metadata documentation and explanation (https://github.com/OpenEnergyPlatform/oemetadata/blob/master/metadata/v141/metadata_key_description.md)", + "dates": "Dates and time must follow the ISO8601 including time zone (YYYY-MM-DD or YYYY-MM-DDThh:mm:ss±hh)", + "units": "Use a space between numbers and units (100 m)", + "languages": "Languages must follow the IETF (BCP47) format (en-GB, en-US, de-DE)", + "licenses": "License name must follow the SPDX License List (https://spdx.org/licenses/)", + "review": "Following the OEP Data Review (https://github.com/OpenEnergyPlatform/data-preprocessing/wiki)", + "none": "If not applicable use (none)", + }, + } + + db.submit_comment( + "'" + json.dumps(meta) + "'", "supply", "egon_power_plants" + ) diff --git a/src/egon/data/datasets/power_plants/pv_ground_mounted.py b/src/egon/data/datasets/power_plants/pv_ground_mounted.py index d4f80374b..8904682b1 100644 --- a/src/egon/data/datasets/power_plants/pv_ground_mounted.py +++ b/src/egon/data/datasets/power_plants/pv_ground_mounted.py @@ -9,7 +9,6 @@ def insert(): def mastr_existing_pv(pow_per_area): - """Import MaStR data from csv-files. Parameters @@ -141,7 +140,6 @@ def mastr_existing_pv(pow_per_area): return mastr def potential_areas(con, join_buffer): - """Import potential areas and choose and prepare areas suitable for PV ground mounted. @@ -293,7 +291,6 @@ def potential_areas(con, join_buffer): return potentials_rora, potentials_agri def select_pot_areas(mastr, potentials_pot): - """Select potential areas where there are existing pv parks (MaStR-data). @@ -343,7 +340,6 @@ def select_pot_areas(mastr, potentials_pot): return (pot_sel, mastr) def build_pv(pv_pot, pow_per_area): - """Build new pv parks in selected potential areas. Parameters @@ -374,7 +370,6 @@ def build_pv(pv_pot, pow_per_area): return pv_pot def adapt_grid_level(pv_pot, max_dist_hv, con): - """Check and if needed adapt grid level of newly built pv parks. Parameters @@ -403,7 +398,6 @@ def adapt_grid_level(pv_pot, max_dist_hv, con): ] if len(pv_pot_mv_to_hv) > 0: - # import data for HV substations sql = "SELECT point, voltage FROM grid.egon_hvmv_substation" @@ -435,7 +429,7 @@ def adapt_grid_level(pv_pot, max_dist_hv, con): pv_pot_mv_to_hv["dist_to_HV"] <= max_dist_hv ] pv_pot_mv_to_hv = pv_pot_mv_to_hv.drop(columns=["dist_to_HV"]) - pv_pot_hv = pv_pot_hv.append(pv_pot_mv_to_hv) + pv_pot_hv = pd.concat([pv_pot_hv, pv_pot_mv_to_hv]) # delete PVs which are now HV from MV dataframe for index, pot in pv_pot_mv_to_hv.iterrows(): @@ -449,12 +443,11 @@ def adapt_grid_level(pv_pot, max_dist_hv, con): ].apply(lambda x: x if x < max_cap_mv else max_cap_mv) pv_pot_mv["voltage_level"] = 5 - pv_pot = pv_pot_mv.append(pv_pot_hv) + pv_pot = pd.concat([pv_pot_mv, pv_pot_hv]) return pv_pot def build_additional_pv(potentials, pv, pow_per_area, con): - """Build additional pv parks if pv parks on selected potential areas do not hit the target value. @@ -535,7 +528,6 @@ def check_target( pow_per_area, con, ): - """Check target value per scenario and per state. Parameters @@ -585,7 +577,6 @@ def check_target( # linear scale farms to meet target if sum of installed capacity is # too high if total_pv_power >= target_power: - scale_factor = target_power / total_pv_power pv_rora_i["installed capacity in kW"] = ( pv_rora_i["installed capacity in kW"] * scale_factor @@ -614,7 +605,6 @@ def check_target( # build new pv parks if sum of installed capacity is below target # value elif total_pv_power < target_power: - rest_cap = target_power - total_pv_power ### @@ -658,7 +648,6 @@ def check_target( # build pv parks on potential areas agriculture if still necessary elif pv_per_distr_i["installed capacity in kW"].sum() < rest_cap: - rest_cap = ( target_power - total_pv_power @@ -711,8 +700,8 @@ def check_target( "and 'Agriculture'." ) - pv_per_distr_i = pv_per_distr_i.append( - pv_per_distr_i_2, ignore_index=True + pv_per_distr_i = pd.concat( + [pv_per_distr_i, pv_per_distr_i_2], ignore_index=True ) # assign grid level to pv_per_distr @@ -776,7 +765,6 @@ def run_methodology( max_dist_hv=20000, show_map=False, ): - """Execute methodology to distribute pv ground mounted. Parameters @@ -889,7 +877,6 @@ def run_methodology( # check target value per state for i in nuts: - target_power = ( target[target["nuts"] == i]["capacity"].iloc[0] * 1000 ) @@ -984,7 +971,7 @@ def run_methodology( pv_agri = pv_agri.append(agri_i) pv_exist = pv_exist.append(exist_i) if len(distr_i) > 0: - pv_per_distr = pv_per_distr.append(distr_i) + pv_per_distr = pd.concat([pv_per_distr, distr_i]) # 2) scenario: eGon100RE @@ -1025,7 +1012,6 @@ def run_methodology( # ### create map to show distribution of installed capacity if show_map == True: - # 1) eGon2035 # get MV grid districts @@ -1155,7 +1141,6 @@ def run_methodology( def insert_pv_parks( pv_rora, pv_agri, pv_exist, pv_per_distr, scenario_name ): - """Write to database. Parameters @@ -1176,8 +1161,8 @@ def insert_pv_parks( # prepare dataframe for integration in supply.egon_power_plants - pv_parks = pv_rora.append( - [pv_agri, pv_exist, pv_per_distr], ignore_index=True + pv_parks = pd.concat( + [pv_rora, pv_agri, pv_exist, pv_per_distr], ignore_index=True ) pv_parks["el_capacity"] = pv_parks["installed capacity in kW"] / 1000 pv_parks.rename(columns={"centroid": "geometry"}, inplace=True) @@ -1307,13 +1292,11 @@ def insert_pv_parks( or pv_per_distr["installed capacity in kW"].sum() > 0 or pv_exist["installed capacity in kW"].sum() > 0 ): - pv_parks = insert_pv_parks( pv_rora, pv_agri, pv_exist, pv_per_distr, "eGon2035" ) else: - pv_parks = gpd.GeoDataFrame() if ( @@ -1322,7 +1305,6 @@ def insert_pv_parks( or pv_per_distr_100RE["installed capacity in kW"].sum() > 0 or pv_exist_100RE["installed capacity in kW"].sum() > 0 ): - pv_parks_100RE = insert_pv_parks( pv_rora_100RE, pv_agri_100RE, @@ -1332,7 +1314,6 @@ def insert_pv_parks( ) else: - pv_parks_100RE = gpd.GeoDataFrame() return pv_parks, pv_parks_100RE diff --git a/src/egon/data/datasets/power_plants/pv_rooftop_buildings.py b/src/egon/data/datasets/power_plants/pv_rooftop_buildings.py index 07c43489d..9faa2fdf7 100644 --- a/src/egon/data/datasets/power_plants/pv_rooftop_buildings.py +++ b/src/egon/data/datasets/power_plants/pv_rooftop_buildings.py @@ -10,10 +10,13 @@ from collections import Counter from functools import wraps from time import perf_counter +import datetime +import json from geoalchemy2 import Geometry from loguru import logger from numpy.random import RandomState, default_rng +from omi.dialects import get_dialect from pyproj.crs.crs import CRS from sqlalchemy import BigInteger, Column, Float, Integer, String from sqlalchemy.dialects.postgresql import HSTORE @@ -28,7 +31,18 @@ ) from egon.data.datasets.power_plants.mastr_db_classes import EgonPowerPlantsPv from egon.data.datasets.scenario_capacities import EgonScenarioCapacities +from egon.data.datasets.scenario_parameters import get_scenario_year from egon.data.datasets.zensus_vg250 import Vg250Gem +from egon.data.metadata import ( + context, + contributors, + generate_resource_fields_from_db_table, + license_dedl, + license_odbl, + meta_metadata, + meta_metadata, + sources, +) engine = db.engine() Base = declarative_base() @@ -51,8 +65,10 @@ Q = 5 # Scenario Data -SCENARIOS = ["eGon2035", "eGon100RE"] +SCENARIOS = config.settings()["egon-data"]["--scenarios"] SCENARIO_TIMESTAMP = { + "status2019": pd.Timestamp("2020-01-01", tz="UTC"), + "status2023": pd.Timestamp("2024-01-01", tz="UTC"), "eGon2035": pd.Timestamp("2035-01-01", tz="UTC"), "eGon100RE": pd.Timestamp("2050-01-01", tz="UTC"), } @@ -461,10 +477,14 @@ def drop_buildings_outside_muns( def egon_building_peak_loads(): - sql = """ + + # use active scenario wich is closest to today + scenario = sorted(SCENARIOS, key=get_scenario_year)[0] + + sql = f""" SELECT building_id FROM demand.egon_building_electricity_peak_loads - WHERE scenario = 'eGon2035' + WHERE scenario = '{scenario}' """ return ( @@ -667,7 +687,6 @@ def allocate_pv( t0 = perf_counter() for count, ags in enumerate(ags_list): - buildings = q_buildings_gdf.loc[q_buildings_gdf.ags == ags] gens = q_mastr_gdf.loc[q_mastr_gdf.ags == ags] @@ -708,7 +727,7 @@ def allocate_pv( ) ) - chosen_buildings = q_buildings.index.append(add_buildings) + chosen_buildings = q_buildings.index.union(add_buildings) else: chosen_buildings = rng.choice( @@ -732,9 +751,9 @@ def allocate_pv( assert len(assigned_buildings) == len(assigned_buildings.gens_id.unique()) - q_mastr_gdf.loc[ - assigned_buildings.gens_id, "building_id" - ] = assigned_buildings.index + q_mastr_gdf.loc[assigned_buildings.gens_id, "building_id"] = ( + assigned_buildings.index + ) assigned_gens = q_mastr_gdf.loc[~q_mastr_gdf.building_id.isna()] @@ -1123,18 +1142,19 @@ def cap_per_bus_id( pandas.DataFrame DataFrame with total rooftop capacity per mv grid. """ - targets = config.datasets()["solar_rooftop"]["targets"] + sources = config.datasets()["solar_rooftop"]["sources"] sql = f""" - SELECT bus as bus_id, control, p_nom as capacity - FROM {targets['generators']['schema']}.{targets['generators']['table']} + SELECT bus_id, SUM(el_capacity) as capacity + FROM {sources['power_plants']['schema']}.{sources['power_plants']['table']} WHERE carrier = 'solar_rooftop' - AND scn_name = '{scenario}' + AND scenario = '{scenario}' + GROUP BY bus_id """ df = db.select_dataframe(sql, index_col="bus_id") - return df.loc[df.control != "Slack"] + return df def determine_end_of_life_gens( @@ -2082,6 +2102,138 @@ class EgonPowerPlantPvRoofBuilding(Base): weather_cell_id = Column(Integer) +def add_metadata(): + schema = "supply" + table = "egon_power_plants_pv_roof_building" + name = f"{schema}.{table}" + deposit_id_mastr = config.datasets()["mastr_new"]["deposit_id"] + deposit_id_data_bundle = config.datasets()["data-bundle"]["sources"][ + "zenodo" + ]["deposit_id"] + + contris = contributors(["kh", "kh"]) + + contris[0]["date"] = "2023-03-16" + + contris[0]["object"] = "metadata" + contris[1]["object"] = "dataset" + + contris[0]["comment"] = "Add metadata to dataset." + contris[1]["comment"] = "Add workflow to generate dataset." + + meta = { + "name": name, + "title": "eGon power plants rooftop solar", + "id": "WILL_BE_SET_AT_PUBLICATION", + "description": ( + "eGon power plants rooftop solar systems allocated to buildings" + ), + "language": "en-US", + "keywords": ["photovoltaik", "solar", "pv", "mastr", "status quo"], + "publicationDate": datetime.date.today().isoformat(), + "context": context(), + "spatial": { + "location": "none", + "extent": "Germany", + "resolution": "building", + }, + "temporal": { + "referenceDate": ( + config.datasets()["mastr_new"]["egon2021_date_max"].split(" ")[ + 0 + ] + ), + "timeseries": {}, + }, + "sources": [ + { + "title": "Data bundle for egon-data", + "description": ( + "Data bundle for egon-data: A transparent and " + "reproducible data processing pipeline for energy " + "system modeling" + ), + "path": ( + "https://zenodo.org/record/" + f"{deposit_id_data_bundle}#.Y_dWM4CZMVM" + ), + "licenses": [license_dedl(attribution="© Cußmann, Ilka")], + }, + { + "title": ("open-MaStR power unit registry for eGo^n project"), + "description": ( + "Data from Marktstammdatenregister (MaStR) data using " + "the data dump from 2022-11-17 for eGon-data." + ), + "path": ( + f"https://zenodo.org/record/{deposit_id_mastr}" + ), + "licenses": [license_dedl(attribution="© Amme, Jonathan")], + }, + sources()["openstreetmap"], + sources()["era5"], + sources()["vg250"], + sources()["egon-data"], + ], + "licenses": [license_odbl("© eGon development team")], + "contributors": contris, + "resources": [ + { + "profile": "tabular-data-resource", + "name": name, + "path": "None", + "format": "PostgreSQL", + "encoding": "UTF-8", + "schema": { + "fields": generate_resource_fields_from_db_table( + schema, + table, + ), + "primaryKey": "index", + }, + "dialect": {"delimiter": "", "decimalSeparator": ""}, + } + ], + "review": {"path": "", "badge": ""}, + "metaMetadata": meta_metadata(), + "_comment": { + "metadata": ( + "Metadata documentation and explanation (https://github." + "com/OpenEnergyPlatform/oemetadata/blob/master/metadata/" + "v141/metadata_key_description.md)" + ), + "dates": ( + "Dates and time must follow the ISO8601 including time " + "zone (YYYY-MM-DD or YYYY-MM-DDThh:mm:ss±hh)" + ), + "units": "Use a space between numbers and units (100 m)", + "languages": ( + "Languages must follow the IETF (BCP47) format (en-GB, " + "en-US, de-DE)" + ), + "licenses": ( + "License name must follow the SPDX License List " + "(https://spdx.org/licenses/)" + ), + "review": ( + "Following the OEP Data Review (https://github.com/" + "OpenEnergyPlatform/data-preprocessing/wiki)" + ), + "none": "If not applicable use (none)", + }, + } + + dialect = get_dialect(f"oep-v{meta_metadata()['metadataVersion'][4:7]}")() + + meta = dialect.compile_and_render(dialect.parse(json.dumps(meta))) + + db.submit_comment( + f"'{json.dumps(meta)}'", + schema, + table, + ) + + def create_scenario_table(buildings_gdf): """Create mapping table pv_unit <-> building for scenario""" EgonPowerPlantPvRoofBuilding.__table__.drop(bind=engine, checkfirst=True) @@ -2095,6 +2247,8 @@ def create_scenario_table(buildings_gdf): index=False, ) + add_metadata() + def add_weather_cell_id(buildings_gdf: gpd.GeoDataFrame) -> gpd.GeoDataFrame: sql = """ @@ -2152,6 +2306,7 @@ def add_bus_ids_sq( grid_districts_gdf = grid_districts(EPSG) mask = buildings_gdf.scenario == "status_quo" + buildings_gdf.loc[mask, "bus_id"] = ( buildings_gdf.loc[mask] .sjoin(grid_districts_gdf, how="left") @@ -2206,6 +2361,16 @@ def pv_rooftop_to_buildings(): mastr_gdf = load_mastr_data() + status_quo = "status2023" + + ts = pd.Timestamp( + config.datasets()["mastr_new"][f"{status_quo}_date_max"], tz="UTC" + ) + + mastr_gdf = mastr_gdf.loc[ + mastr_gdf.commissioning_date <= ts + ] + buildings_gdf = load_building_data() desagg_mastr_gdf, desagg_buildings_gdf = allocate_to_buildings( @@ -2213,9 +2378,10 @@ def pv_rooftop_to_buildings(): ) all_buildings_gdf = ( - desagg_mastr_gdf.assign(scenario="status_quo") + desagg_mastr_gdf.assign(scenario=status_quo) .reset_index() .rename(columns={"geometry": "geom"}) + .set_geometry("geom") ) scenario_buildings_gdf = all_buildings_gdf.copy() @@ -2223,16 +2389,33 @@ def pv_rooftop_to_buildings(): cap_per_bus_id_df = pd.DataFrame() for scenario in SCENARIOS: - logger.debug(f"Desaggregating scenario {scenario}.") - ( - scenario_buildings_gdf, - cap_per_bus_id_scenario_df, - ) = allocate_scenarios( # noqa: F841 - desagg_mastr_gdf, - desagg_buildings_gdf, - scenario_buildings_gdf, - scenario, - ) + if scenario == status_quo: + continue + elif "status" in scenario: + ts = pd.Timestamp( + config.datasets()["mastr_new"][f"{scenario}_date_max"], tz="UTC" + ) + + scenario_buildings_gdf = scenario_buildings_gdf.loc[ + scenario_buildings_gdf.commissioning_date <= ts + ] + + else: + logger.debug(f"Desaggregating scenario {scenario}.") + + ( + scenario_buildings_gdf, + cap_per_bus_id_scenario_df, + ) = allocate_scenarios( # noqa: F841 + desagg_mastr_gdf, + desagg_buildings_gdf, + scenario_buildings_gdf, + scenario, + ) + + cap_per_bus_id_df = pd.concat( + [cap_per_bus_id_df, cap_per_bus_id_scenario_df] + ) all_buildings_gdf = gpd.GeoDataFrame( pd.concat( @@ -2242,10 +2425,6 @@ def pv_rooftop_to_buildings(): geometry="geom", ) - cap_per_bus_id_df = pd.concat( - [cap_per_bus_id_df, cap_per_bus_id_scenario_df] - ) - # add weather cell all_buildings_gdf = add_weather_cell_id(all_buildings_gdf) diff --git a/src/egon/data/datasets/power_plants/wind_farms.py b/src/egon/data/datasets/power_plants/wind_farms.py index 9d57d05c9..89c18a14f 100755 --- a/src/egon/data/datasets/power_plants/wind_farms.py +++ b/src/egon/data/datasets/power_plants/wind_farms.py @@ -101,8 +101,8 @@ def insert(): source, fed_state, ) - summary_t = summary_t.append(summary_state) - farms = farms.append(wind_farms_state) + summary_t = pd.concat([summary_t, summary_state]) + farms = pd.concat([farms, wind_farms_state]) generate_map() @@ -347,7 +347,7 @@ def match_district_se(x): wf_mv_to_hv = wf_mv_to_hv.drop(columns=["dist_to_HV"]) wf_mv_to_hv["voltage"] = "Hochspannung" - wf_hv = wf_hv.append(wf_mv_to_hv) + wf_hv = pd.concat([wf_hv, wf_mv_to_hv]) wf_mv = wf_mv[ (wf_mv["dist_to_HV"] > max_dist_hv) | (wf_mv["inst capacity [MW]"] < max_power_mv) @@ -362,7 +362,7 @@ def match_district_se(x): lambda x: x if x < max_power_mv else max_power_mv ) - wind_farms = wf_hv.append(wf_mv) + wind_farms = pd.concat([wf_hv, wf_mv]) # Adjust the total installed capacity to the scenario total_wind_power = ( @@ -376,14 +376,22 @@ def match_district_se(x): wf_hv["inst capacity [MW]"] = ( wf_hv["inst capacity [MW]"] * scale_factor ) - wind_farms = wf_hv.append(wf_mv) - summary = summary.append( - { - "state": fed_state, - "target": target_power, - "from existin WF": wind_farms["inst capacity [MW]"].sum(), - "MV districts": 0, - }, + wind_farms = pd.concat([wf_hv, wf_mv]) + summary = pd.concat( + [ + summary, + pd.DataFrame( + index=[summary.index.max() + 1], + data={ + "state": fed_state, + "target": target_power, + "from existin WF": wind_farms[ + "inst capacity [MW]" + ].sum(), + "MV districts": 0, + }, + ), + ], ignore_index=True, ) else: @@ -409,16 +417,24 @@ def match_district_se(x): scale_factor = (target_power - total_wind_power) / total_new_area extra_wf["inst capacity [MW]"] = extra_wf["area [km²]"] * scale_factor extra_wf["voltage"] = "Hochspannung" - summary = summary.append( - { - "state": fed_state, - "target": target_power, - "from existin WF": wind_farms["inst capacity [MW]"].sum(), - "MV districts": extra_wf["inst capacity [MW]"].sum(), - }, + summary = pd.concat( + [ + summary, + pd.DataFrame( + index=[summary.index.max() + 1], + data={ + "state": fed_state, + "target": target_power, + "from existin WF": wind_farms[ + "inst capacity [MW]" + ].sum(), + "MV districts": extra_wf["inst capacity [MW]"].sum(), + }, + ), + ], ignore_index=True, ) - wind_farms = wind_farms.append(extra_wf, ignore_index=True) + wind_farms = pd.concat([wind_farms, extra_wf], ignore_index=True) # Use Definition of thresholds for voltage level assignment wind_farms["voltage_level"] = 0 diff --git a/src/egon/data/datasets/power_plants/wind_offshore.py b/src/egon/data/datasets/power_plants/wind_offshore.py index ba78778c8..85aa2083e 100644 --- a/src/egon/data/datasets/power_plants/wind_offshore.py +++ b/src/egon/data/datasets/power_plants/wind_offshore.py @@ -1,18 +1,140 @@ from pathlib import Path +import logging from shapely.geometry import Point import geopandas as gpd -import numpy as np import pandas as pd from egon.data import db import egon.data.config +def map_id_bus(scenario): + # Import manually generated list of wind offshore farms with their + # connection points (OSM_id) + if scenario == "eGon2035": + id_bus = { + "Büttel": "136034396", + "Heide/West": "603661085", + "Suchraum Gemeinden Ibbenbüren/Mettingen/Westerkappeln": "114319248", + "Suchraum Zensenbusch": "76185022", + "Rommerskirchen": "24839976", + "Oberzier": "26593929", + "Garrel/Ost": "23837631", + "Diele": "177829920", + "Dörpen/West": "142487746", + "Emden/Borßum": "34835258", + "Emden/Ost": "34835258", + "Hagermarsch": "79316833", + "Hanekenfähr": "61918154", + "Inhausen": "29420322", + "Unterweser": "32076853", + "Wehrendorf": "33411203", + "Wilhelmshaven 2": "23744346", + "Rastede": "23837631", + "Bentwisch": "32063539", + "Lubmin": "460134233", + "Suchraum Gemeinde Papendorf": "32063539", + "Suchraum Gemeinden Brünzow/Kemnitz": "460134233", + "inhausen": "29420322", + "Cloppenburg": "50643382", + } + elif "status" in scenario: + year = int(scenario[-4:]) + + id_bus = { + "UW Inhausen": "29420322", + "UW Bentwisch": "32063539", + "UW Emden / Borssum": "34835258", + "UW Emden Ost": "34835258", + "UW Cloppenburg": "50643382", + "UW Hagermarsch": "79316833", + "UW Büttel": "136034396", + "UW Dörpen West": "142487746", + "UW Diele": "177829920", + "UW Lubmin": "460134233", + } + + if year >= 2023: + # No update needed as no new stations used for offshore wind + # between 2019 and 2023 + pass + + # TODO: If necessary add new stations when generating status quo > 2023 + + else: + id_bus = {} + + return id_bus + + +def assign_ONEP_areas(): + return { + "Büttel": "NOR-4-1", + "Heide/West": "NOR-10-2", + "Suchraum Gemeinden Ibbenbüren/Mettingen/Westerkappeln": "NOR-9-2", + "Suchraum Zensenbusch": "NOR-7-1", + "Rommerskirchen": "NOR-7-1", + "Oberzier": "NOR-7-1", + "Garrel/Ost": "NOR-7-1", + "Diele": "NOR-6-1", + "Dörpen/West": "NOR-2-2", + "Emden/Borßum": "NOR-0-1", + "Emden/Ost": "NOR-3-3", + "Hagermarsch": "NOR-2-1", + "Hanekenfähr": "NOR-6-3", + "Inhausen": "NOR-0-2", + "Unterweser": "NOR-9-1", + "Wehrendorf": "NOR-7-1", + "Wilhelmshaven 2": "NOR-11-1", + "Rastede": "NOR-7-1", + "Bentwisch": "OST-3-1", + "Lubmin": "OST-1-1", + "Suchraum Gemeinde Papendorf": "OST-7-1", + "Suchraum Gemeinden Brünzow/Kemnitz": "OST-1-4", + "inhausen": "NOR-0-2", + "Cloppenburg": "NOR-4-1", + } + + +def map_ONEP_areas(): + return { + "NOR-0-1": Point(6.5, 53.6), + "NOR-0-2": Point(8.07, 53.76), + "NOR-1": Point(6.21, 54.06), + "NOR-1-1": Point(6.21, 54.06), + "NOR-2-1": Point(6.54, 53.99), + "NOR-2-2": Point(6.54, 53.99), + "NOR-2-3": Point(6.54, 53.99), + "NOR-3-1": Point(6.95, 54.02), + "NOR-3-3": Point(6.95, 54.02), + "NOR-4-1": Point(7.70, 54.44), + "NOR-4-2": Point(7.70, 54.44), + "NOR-5-1": Point(7.21, 55.14), + "NOR-6-1": Point(5.92, 54.30), + "NOR-6-2": Point(5.92, 54.30), + "NOR-6-3": Point(5.92, 54.30), + "NOR-7": Point(6.22, 54.32), + "NOR-7-1": Point(6.22, 54.32), + "NOR-8-1": Point(6.35, 54.48), + "NOR-9-1": Point(5.75, 54.5), + "NOR-9-2": Point(5.75, 54.5), + "NOR-10-2": Point(6, 54.75), + "NOR-11-1": Point(6.5, 54.75), + "OST-1-1": Point(14.09, 54.82), + "OST-1-2": Point(14.09, 54.82), + "OST-1-3": Point(14.09, 54.82), + "OST-1-4": Point(14.09, 54.82), + "OST-2": Point(13.86, 54.83), + "OST-3-1": Point(13.16, 54.98), + "OST-3-2": Point(13.16, 54.98), + "OST-7-1": Point(12.25, 54.5), + } + + def insert(): """ Include the offshore wind parks in egon-data. - locations and installed capacities based on: NEP2035_V2021_scnC2035 Parameters ---------- @@ -21,206 +143,213 @@ def insert(): # Read file with all required input/output tables' names cfg = egon.data.config.datasets()["power_plants"] - # load NEP2035_V2021_scnC2035 file - offshore_path = ( - Path(".") - / "data_bundle_egon_data" - / "nep2035_version2021" - / cfg["sources"]["nep_2035"] - ) - - offshore = pd.read_excel( - offshore_path, - sheet_name="WInd_Offshore_NEP", - usecols=[ - "Netzverknuepfungspunkt", - "Spannungsebene in kV", - "C 2035", - "B 2040 ", - ], - ) - offshore.dropna(subset=["Netzverknuepfungspunkt"], inplace=True) + scenarios = egon.data.config.settings()["egon-data"]["--scenarios"] + + for scenario in scenarios: + # Delete previous generators + db.execute_sql( + f""" + DELETE FROM {cfg['target']['schema']}.{cfg['target']['table']} + WHERE carrier = 'wind_offshore' + AND scenario = '{scenario}' + """ + ) + + # load file + if scenario == "eGon2035": + offshore_path = ( + Path(".") + / "data_bundle_egon_data" + / "nep2035_version2021" + / cfg["sources"]["nep_2035"] + ) + + offshore = pd.read_excel( + offshore_path, + sheet_name="WInd_Offshore_NEP", + usecols=[ + "Netzverknuepfungspunkt", + "Spannungsebene in kV", + "C 2035", + ], + ) + offshore.dropna(subset=["Netzverknuepfungspunkt"], inplace=True) + offshore.rename(columns={"C 2035": "el_capacity"}, inplace=True) + + elif scenario == "eGon100RE": + offshore_path = ( + Path(".") + / "data_bundle_egon_data" + / "nep2035_version2021" + / cfg["sources"]["nep_2035"] + ) + + offshore = pd.read_excel( + offshore_path, + sheet_name="WInd_Offshore_NEP", + usecols=[ + "Netzverknuepfungspunkt", + "Spannungsebene in kV", + "B 2040 ", + ], + ) + offshore.dropna(subset=["Netzverknuepfungspunkt"], inplace=True) + + elif "status" in scenario: + year = int(scenario[-4:]) + + offshore_path = ( + Path(".") + / "data_bundle_egon_data" + / "wind_offshore_status2019" + / cfg["sources"]["wind_offshore_status2019"] + ) + offshore = pd.read_excel( + offshore_path, + sheet_name="wind_offshore", + usecols=[ + "Name ONEP/NEP", + "NVP", + "Spannung [kV]", + "Inbetriebnahme", + "Kapazität Gesamtsystem [MW]", + ], + ) + offshore.dropna(subset=["Name ONEP/NEP"], inplace=True) + offshore.rename( + columns={ + "NVP": "Netzverknuepfungspunkt", + "Spannung [kV]": "Spannungsebene in kV", + "Kapazität Gesamtsystem [MW]": "el_capacity", + }, + inplace=True, + ) + offshore = offshore[offshore["Inbetriebnahme"] <= year] - # Import manually generated list of wind offshore farms with their - # connection points (OSM_id) - id_bus = { - "Büttel": "136034396", - "Heide/West": "603661085", - "Suchraum Gemeinden Ibbenbüren/Mettingen/Westerkappeln": "114319248", - "Suchraum Zensenbusch": "76185022", - "Rommerskirchen": "24839976", - "Oberzier": "26593929", - "Garrel/Ost": "23837631", - "Diele": "177829920", - "Dörpen/West": "142487746", - "Emden/Borßum": "34835258", - "Emden/Ost": "34835258", - "Hagermarsch": "79316833", - "Hanekenfähr": "61918154", - "Inhausen": "29420322", - "Unterweser": "32076853", - "Wehrendorf": "33411203", - "Wilhelmshaven 2": "23744346", - "Rastede": "23837631", - "Bentwisch": "32063539", - "Lubmin": "460134233", - "Suchraum Gemeinde Papendorf": "32063539", - "Suchraum Gemeinden Brünzow/Kemnitz": "460134233", - } - w_id = { - "Büttel": "16331", - "Heide/West": "16516", - "Suchraum Gemeinden Ibbenbüren/Mettingen/Westerkappeln": "16326", - "Suchraum Zensenbusch": "16139", - "Rommerskirchen": "16139", - "Oberzier": "16139", - "Garrel/Ost": "16139", - "Diele": "16138", - "Dörpen/West": "15952", - "Emden/Borßum": "15762", - "Emden/Ost": "16140", - "Hagermarsch": "15951", - "Hanekenfähr": "16139", - "Inhausen": "15769", - "Unterweser": "16517", - "Wehrendorf": "16139", - "Wilhelmshaven 2": "16517", - "Rastede": "16139", - "Bentwisch": "16734", - "Lubmin": "16548", - "Suchraum Gemeinde Papendorf": "16352", - "Suchraum Gemeinden Brünzow/Kemnitz": "16548", - } - - # Match wind offshore table with the corresponding OSM_id - offshore["osm_id"] = offshore["Netzverknuepfungspunkt"].map(id_bus) - - # Connect to the data-base - con = db.engine() - - # Import table with all the busses of the grid - sql = f""" - SELECT bus_i as bus_id, geom as point, CAST(osm_substation_id AS text) - as osm_id FROM {cfg["sources"]["buses_data"]} - """ - - busses = gpd.GeoDataFrame.from_postgis( - sql, con, crs="EPSG:4326", geom_col="point" - ) - - # Drop NANs in column osm_id - busses.dropna(subset=["osm_id"], inplace=True) - - # Create columns for bus_id and geometry in the offshore df - offshore["bus_id"] = np.nan - offshore["geom"] = Point(0, 0) - - # Match bus_id and geometry - for index, wind_park in offshore.iterrows(): - if len(busses[busses["osm_id"] == wind_park["osm_id"]].index) > 0: - bus_ind = busses[busses["osm_id"] == wind_park["osm_id"]].index[0] - offshore.at[index, "bus_id"] = busses.at[bus_ind, "bus_id"] - offshore.at[index, "geom"] = busses.at[bus_ind, "point"] else: - print(f'Wind offshore farm not found: {wind_park["osm_id"]}') - - offshore["weather_cell_id"] = offshore["Netzverknuepfungspunkt"].map(w_id) - offshore["weather_cell_id"] = offshore["weather_cell_id"].apply(int) - # Drop offshore wind farms without found connexion point - offshore.dropna(subset=["bus_id"], inplace=True) - - # Assign voltage levels to wind offshore parks - offshore["voltage_level"] = np.nan - offshore.loc[ - offshore[offshore["Spannungsebene in kV"] == 110].index, - "voltage_level", - ] = 3 - offshore.loc[ - offshore[offshore["Spannungsebene in kV"] > 110].index, "voltage_level" - ] = 1 - - # Delete unnecessary columns - offshore.drop( - [ - "Netzverknuepfungspunkt", - "Spannungsebene in kV", - "osm_id", - ], - axis=1, - inplace=True, - ) - - # Assign static values - offshore["carrier"] = "wind_offshore" - - # Create wind farms for the different scenarios - offshore_2035 = offshore.drop(columns=["B 2040 "]).copy() - offshore_2035["scenario"] = "eGon2035" - offshore_2035.rename(columns={"C 2035": "el_capacity"}, inplace=True) - - offshore_100RE = offshore.drop(columns=["C 2035"]).copy() - offshore_100RE["scenario"] = "eGon100RE" - offshore_100RE.rename(columns={"B 2040 ": "el_capacity"}, inplace=True) - - # Import capacity targets for wind_offshore per scenario - sql = f""" - SELECT * FROM {cfg["sources"]["capacities"]} - WHERE scenario_name = 'eGon100RE' AND - carrier = 'wind_offshore' - """ - capacities = pd.read_sql(sql, con) - cap_100RE = capacities.capacity.sum() - - # Scale capacities to match target - scale_factor = cap_100RE / offshore_100RE.el_capacity.sum() - offshore_100RE["el_capacity"] = ( - offshore_100RE["el_capacity"] * scale_factor - ) - - # Join power plants from the different scenarios - offshore = pd.concat([offshore_2035, offshore_100RE], axis=0) - - # convert column "bus_id" and "voltage_level" to integer - offshore["bus_id"] = offshore["bus_id"].apply(int) - offshore["voltage_level"] = offshore["voltage_level"].apply(int) - - # Delete, in case of existing, previous wind offshore parks - - db.execute_sql( - f""" - DELETE FROM {cfg['target']['schema']}.{cfg['target']['table']} - WHERE carrier IN ('wind_offshore') - """ - ) - - # Look for the maximum id in the table egon_power_plants - sql = ( - "SELECT MAX(id) FROM " - + cfg["target"]["schema"] - + "." - + cfg["target"]["table"] - ) - max_id = pd.read_sql(sql, con) - max_id = max_id["max"].iat[0] - if max_id is None: - ini_id = 1 - else: - ini_id = int(max_id + 1) - - offshore = gpd.GeoDataFrame(offshore, geometry="geom", crs=4326) - - # write_table in egon-data database: - # Reset index - offshore.index = pd.RangeIndex( - start=ini_id, stop=ini_id + len(offshore), name="id" - ) - - # Insert into database - offshore.reset_index().to_postgis( - cfg["target"]["table"], - schema=cfg["target"]["schema"], - con=db.engine(), - if_exists="append", - ) - - return "Off shore wind farms successfully created" + raise ValueError(f"{scenario=} is not valid.") + + id_bus = map_id_bus(scenario) + + # Match wind offshore table with the corresponding OSM_id + offshore["osm_id"] = offshore["Netzverknuepfungspunkt"].map(id_bus) + + buses = db.select_geodataframe( + f""" + SELECT bus_i as bus_id, base_kv, geom as point, CAST(osm_substation_id AS text) + as osm_id FROM {cfg["sources"]["buses_data"]} + """, + epsg=4326, + geom_col="point", + ) + + # Drop NANs in column osm_id + buses.dropna(subset=["osm_id"], inplace=True) + + # Create columns for bus_id and geometry in the offshore df + offshore["bus_id"] = pd.NA + offshore["geom"] = Point(0, 0) + + # Match bus_id + for index, wind_park in offshore.iterrows(): + if not buses[ + (buses["osm_id"] == wind_park["osm_id"]) + & (buses["base_kv"] == wind_park["Spannungsebene in kV"]) + ].empty: + bus_ind = buses[buses["osm_id"] == wind_park["osm_id"]].index[ + 0 + ] + offshore.at[index, "bus_id"] = buses.at[bus_ind, "bus_id"] + else: + print(f'Wind offshore farm not found: {wind_park["osm_id"]}') + + offshore.dropna(subset=["bus_id"], inplace=True) + + # Overwrite geom for status2019 parks + if scenario in ["eGon2035", "eGon100RE"]: + offshore["Name ONEP/NEP"] = offshore["Netzverknuepfungspunkt"].map( + assign_ONEP_areas() + ) + + offshore["geom"] = offshore["Name ONEP/NEP"].map(map_ONEP_areas()) + offshore["weather_cell_id"] = pd.NA + + offshore.drop(["Name ONEP/NEP"], axis=1, inplace=True) + + if "status" in scenario: + offshore.drop(["Inbetriebnahme"], axis=1, inplace=True) + + # Scale capacities for eGon100RE + if scenario == "eGon100RE": + # Import capacity targets for wind_offshore per scenario + cap_100RE = db.select_dataframe( + f""" + SELECT SUM(capacity) + FROM {cfg["sources"]["capacities"]} + WHERE scenario_name = 'eGon100RE' AND + carrier = 'wind_offshore' + """ + ) + + # Scale capacities to match target + scale_factor = cap_100RE / offshore.el_capacity.sum() + offshore["el_capacity"] *= scale_factor + + # Assign voltage levels to wind offshore parks + offshore["voltage_level"] = 0 + offshore.loc[ + offshore[offshore["Spannungsebene in kV"] == 110].index, + "voltage_level", + ] = 3 + offshore.loc[ + offshore[offshore["Spannungsebene in kV"] > 110].index, + "voltage_level", + ] = 1 + + # Delete unnecessary columns + offshore.drop( + [ + "Netzverknuepfungspunkt", + "Spannungsebene in kV", + "osm_id", + ], + axis=1, + inplace=True, + ) + + # Set static columns + offshore["carrier"] = "wind_offshore" + offshore["scenario"] = scenario + + offshore = gpd.GeoDataFrame(offshore, geometry="geom", crs=4326) + + # Look for the maximum id in the table egon_power_plants + next_id = ( + db.select_dataframe( + "SELECT MAX(id) FROM " + + cfg["target"]["schema"] + + "." + + cfg["target"]["table"] + ).iloc[0, 0] + + 1 + ) + + # Reset index + offshore.index = pd.RangeIndex( + start=next_id, stop=next_id + len(offshore), name="id" + ) + + # Insert into database + offshore.reset_index().to_postgis( + cfg["target"]["table"], + schema=cfg["target"]["schema"], + con=db.engine(), + if_exists="append", + ) + + logging.info( + f""" + {len(offshore)} wind_offshore generators with a total installed capacity of + {offshore['el_capacity'].sum()}MW were inserted into the db + """ + ) diff --git a/src/egon/data/datasets/pypsaeursec/__init__.py b/src/egon/data/datasets/pypsaeursec/__init__.py index 7d2a43897..2bfb68b51 100755 --- a/src/egon/data/datasets/pypsaeursec/__init__.py +++ b/src/egon/data/datasets/pypsaeursec/__init__.py @@ -16,7 +16,7 @@ import pypsa import yaml -from egon.data import __path__, db, logger +from egon.data import __path__, db, logger, config from egon.data.datasets import Dataset from egon.data.datasets.scenario_parameters import get_sector_parameters import egon.data.config @@ -24,7 +24,6 @@ def run_pypsa_eur_sec(): - cwd = Path(".") filepath = cwd / "run-pypsa-eur-sec" filepath.mkdir(parents=True, exist_ok=True) @@ -132,7 +131,6 @@ def run_pypsa_eur_sec(): def read_network(): - # Set execute_pypsa_eur_sec to False until optional task is implemented execute_pypsa_eur_sec = False cwd = Path(".") @@ -266,7 +264,6 @@ def clean_database(): def neighbor_reduction(): - network = read_network() network.links.drop("pipe_retrofit", axis="columns", inplace=True) @@ -505,7 +502,7 @@ def neighbor_reduction(): neighbor_gens.index += db.next_etrago_id("generator") for i in neighbor_gens_t.columns: - new_index = neighbor_gens[neighbor_gens["name"] == i].index + new_index = neighbor_gens[neighbor_gens["Generator"] == i].index neighbor_gens_t.rename(columns={i: new_index[0]}, inplace=True) # loads @@ -523,7 +520,7 @@ def neighbor_reduction(): neighbor_loads.index += db.next_etrago_id("load") for i in neighbor_loads_t.columns: - new_index = neighbor_loads[neighbor_loads["index"] == i].index + new_index = neighbor_loads[neighbor_loads["Load"] == i].index neighbor_loads_t.rename(columns={i: new_index[0]}, inplace=True) # stores @@ -540,7 +537,7 @@ def neighbor_reduction(): neighbor_stores.index += db.next_etrago_id("store") for i in neighbor_stores_t.columns: - new_index = neighbor_stores[neighbor_stores["name"] == i].index + new_index = neighbor_stores[neighbor_stores["Store"] == i].index neighbor_stores_t.rename(columns={i: new_index[0]}, inplace=True) # storage_units @@ -563,7 +560,9 @@ def neighbor_reduction(): neighbor_storage.index += db.next_etrago_id("storage") for i in neighbor_storage_t.columns: - new_index = neighbor_storage[neighbor_storage["name"] == i].index + new_index = neighbor_storage[ + neighbor_storage["StorageUnit"] == i + ].index neighbor_storage_t.rename(columns={i: new_index[0]}, inplace=True) # Connect to local database @@ -588,10 +587,19 @@ def neighbor_reduction(): c_neighbors = pd.concat([coordinates, c_neighbors], axis=1).set_index( "new_index", drop=False ) - non_AC_neighbors = non_AC_neighbors.append(c_neighbors) - neighbors = neighbors[neighbors.carrier == "AC"].append(non_AC_neighbors) + non_AC_neighbors = pd.concat([non_AC_neighbors, c_neighbors]) + neighbors = pd.concat( + [neighbors[neighbors.carrier == "AC"], non_AC_neighbors] + ) - for i in ["new_index", "control", "generator", "location", "sub_network"]: + for i in [ + "new_index", + "control", + "generator", + "location", + "sub_network", + "unit", + ]: neighbors = neighbors.drop(i, axis=1) # Add geometry column @@ -632,7 +640,7 @@ def lines_to_etrago(neighbor_lines=neighbor_lines, scn="eGon100RE"): neighbor_lines["s_nom"] = neighbor_lines["s_nom_min"] for i in [ - "name", + "Line", "x_pu_eff", "r_pu_eff", "sub_network", @@ -671,8 +679,8 @@ def lines_to_etrago(neighbor_lines=neighbor_lines, scn="eGon100RE"): index_label="line_id", ) - lines_to_etrago(neighbor_lines=neighbor_lines, scn="eGon100RE") - lines_to_etrago(neighbor_lines=neighbor_lines, scn="eGon2035") + for scn in config.settings()["egon-data"]["--scenarios"]: + lines_to_etrago(neighbor_lines=neighbor_lines, scn=scn) def links_to_etrago(neighbor_links, scn="eGon100RE", extendable=True): """Prepare and write neighboring crossborder links to eTraGo table @@ -710,7 +718,7 @@ def links_to_etrago(neighbor_links, scn="eGon100RE", extendable=True): if extendable is True: neighbor_links = neighbor_links.drop( columns=[ - "name", + "Link", "geometry", "tags", "under_construction", @@ -725,6 +733,17 @@ def links_to_etrago(neighbor_links, scn="eGon100RE", extendable=True): "lifetime", "p_nom_opt", "pipe_retrofit", + "committable", + "start_up_cost", + "shut_down_cost", + "min_up_time", + "min_down_time", + "up_time_before", + "down_time_before", + "ramp_limit_up", + "ramp_limit_down", + "ramp_limit_start_up", + "ramp_limit_shut_down", ], errors="ignore", ) @@ -732,7 +751,7 @@ def links_to_etrago(neighbor_links, scn="eGon100RE", extendable=True): elif extendable is False: neighbor_links = neighbor_links.drop( columns=[ - "name", + "Link", "geometry", "tags", "under_construction", @@ -748,6 +767,17 @@ def links_to_etrago(neighbor_links, scn="eGon100RE", extendable=True): "p_nom", "p_nom_extendable", "pipe_retrofit", + "committable", + "start_up_cost", + "shut_down_cost", + "min_up_time", + "min_down_time", + "up_time_before", + "down_time_before", + "ramp_limit_up", + "ramp_limit_start_up", + "ramp_limit_down", + "ramp_limit_shut_down", ], errors="ignore", ) @@ -832,7 +862,11 @@ def links_to_etrago(neighbor_links, scn="eGon100RE", extendable=True): extendable=False, ) - links_to_etrago(neighbor_links[neighbor_links.carrier == "DC"], "eGon2035") + for scn in config.settings()["egon-data"]["--scenarios"]: + if scn != "eGon100RE": + links_to_etrago( + neighbor_links[neighbor_links.carrier == "DC"], scn + ) # prepare neighboring generators for etrago tables neighbor_gens["scn_name"] = "eGon100RE" @@ -855,7 +889,14 @@ def links_to_etrago(neighbor_links, scn="eGon100RE", extendable=True): inplace=True, ) - for i in ["name", "weight", "lifetime", "p_set", "q_set", "p_nom_opt"]: + for i in [ + "Generator", + "weight", + "lifetime", + "p_set", + "q_set", + "p_nom_opt", + ]: neighbor_gens = neighbor_gens.drop(i, axis=1) neighbor_gens.to_sql( @@ -886,7 +927,7 @@ def links_to_etrago(neighbor_links, scn="eGon100RE", extendable=True): ) neighbor_loads = neighbor_loads.drop( - columns=["index"], + columns=["Load"], errors="ignore", ) @@ -927,8 +968,16 @@ def links_to_etrago(neighbor_links, scn="eGon100RE", extendable=True): "carrier", ] = "H2_overground" - for i in ["name", "p_set", "q_set", "e_nom_opt", "lifetime"]: - neighbor_stores = neighbor_stores.drop(i, axis=1) + for i in [ + "Store", + "p_set", + "q_set", + "e_nom_opt", + "lifetime", + "e_initial_per_period", + "e_cyclic_per_period", + ]: + neighbor_stores = neighbor_stores.drop(i, axis=1, errors="ignore") for c in ["H2_underground", "H2_overground"]: neighbor_stores.loc[ @@ -955,8 +1004,13 @@ def links_to_etrago(neighbor_links, scn="eGon100RE", extendable=True): {"PHS": "pumped_hydro", "hydro": "reservoir"}, inplace=True ) - for i in ["name", "p_nom_opt"]: - neighbor_storage = neighbor_storage.drop(i, axis=1) + for i in [ + "StorageUnit", + "p_nom_opt", + "state_of_charge_initial_per_period", + "cyclic_state_of_charge_per_period", + ]: + neighbor_storage = neighbor_storage.drop(i, axis=1, errors="ignore") neighbor_storage.to_sql( "egon_etrago_storage", diff --git a/src/egon/data/datasets/renewable_feedin.py b/src/egon/data/datasets/renewable_feedin.py index ea81d3330..2858c77cb 100644 --- a/src/egon/data/datasets/renewable_feedin.py +++ b/src/egon/data/datasets/renewable_feedin.py @@ -2,6 +2,9 @@ Central module containing all code dealing with processing era5 weather data. """ +import datetime +import json +import time from sqlalchemy import Column, ForeignKey, Integer from sqlalchemy.ext.declarative import declarative_base import geopandas as gpd @@ -10,8 +13,14 @@ from egon.data import db from egon.data.datasets import Dataset -from egon.data.datasets.era5 import EgonEra5Cells, import_cutout +from egon.data.datasets.era5 import EgonEra5Cells, EgonRenewableFeedIn, import_cutout from egon.data.datasets.scenario_parameters import get_sector_parameters +from egon.data.metadata import ( + context, + license_ccby, + meta_metadata, + sources, +) from egon.data.datasets.zensus_vg250 import DestatisZensusPopulationPerHa import egon.data.config @@ -235,7 +244,7 @@ def feedin_per_turbine(): # Select weather data for Germany cutout = import_cutout(boundary="Germany") - gdf = gpd.GeoDataFrame(geometry=cutout.grid_cells(), crs=4326) + gdf = gpd.GeoDataFrame(geometry=cutout.grid.geometry, crs=4326) # Calculate feedin-timeseries for E-141 # source: @@ -276,7 +285,7 @@ def feedin_per_turbine(): ), } ts_e141 = cutout.wind( - turbine_e141, per_unit=True, shapes=cutout.grid_cells() + turbine_e141, per_unit=True, shapes=cutout.grid.geometry ) gdf["E-141"] = ts_e141.to_pandas().transpose().values.tolist() @@ -320,7 +329,7 @@ def feedin_per_turbine(): ), } ts_e126 = cutout.wind( - turbine_e126, per_unit=True, shapes=cutout.grid_cells() + turbine_e126, per_unit=True, shapes=cutout.grid.geometry ) gdf["E-126"] = ts_e126.to_pandas().transpose().values.tolist() @@ -637,3 +646,103 @@ def mapping_zensus_weather(): orient="records" ), ) + + +def add_metadata(): + """Add metdata to supply.egon_era5_renewable_feedin + + Returns + ------- + None. + + """ + + # Import column names and datatypes + fields = [ + { + "description": "Weather cell index", + "name": "w_id", + "type": "integer", + "unit": "none", + }, + { + "description": "Weather year", + "name": "weather_year", + "type": "integer", + "unit": "none", + }, + { + "description": "Energy carrier", + "name": "carrier", + "type": "string", + "unit": "none", + }, + { + "description": "Weather-dependent feedin timeseries", + "name": "feedin", + "type": "array", + "unit": "p.u.", + }, + ] + + meta = { + "name": "supply.egon_era5_renewable_feedin", + "title": "eGon feedin timeseries for RES", + "id": "WILL_BE_SET_AT_PUBLICATION", + "description": "Weather-dependent feedin timeseries for RES", + "language": ["EN"], + "publicationDate": datetime.date.today().isoformat(), + "context": context(), + "spatial": { + "location": None, + "extent": "Germany", + "resolution": None, + }, + "sources": [ + sources()["era5"], + sources()["vg250"], + sources()["egon-data"], + ], + "licenses": [ + license_ccby( + "© Bundesamt für Kartographie und Geodäsie 2020 (Daten verändert); " + "© Copernicus Climate Change Service (C3S) Climate Data Store " + "© Jonathan Amme, Clara Büttner, Ilka Cußmann, Julian Endres, Carlos Epia, Stephan Günther, Ulf Müller, Amélia Nadal, Guido Pleßmann, Francesco Witte", + ) + ], + "contributors": [ + { + "title": "Clara Büttner", + "email": "http://github.com/ClaraBuettner", + "date": time.strftime("%Y-%m-%d"), + "object": None, + "comment": "Imported data", + }, + ], + "resources": [ + { + "profile": "tabular-data-resource", + "name": "supply.egon_scenario_capacities", + "path": None, + "format": "PostgreSQL", + "encoding": "UTF-8", + "schema": { + "fields": fields, + "primaryKey": ["index"], + "foreignKeys": [], + }, + "dialect": {"delimiter": None, "decimalSeparator": "."}, + } + ], + "metaMetadata": meta_metadata(), + } + + # Create json dump + meta_json = "'" + json.dumps(meta) + "'" + + # Add metadata as a comment to the table + db.submit_comment( + meta_json, + EgonRenewableFeedIn.__table__.schema, + EgonRenewableFeedIn.__table__.name, + ) diff --git a/src/egon/data/datasets/scenario_capacities.py b/src/egon/data/datasets/scenario_capacities.py index 112cde828..9f45d44d9 100755 --- a/src/egon/data/datasets/scenario_capacities.py +++ b/src/egon/data/datasets/scenario_capacities.py @@ -3,6 +3,9 @@ """ from pathlib import Path +import datetime +import json +import time from sqlalchemy import Column, Float, Integer, String from sqlalchemy.ext.declarative import declarative_base @@ -13,10 +16,16 @@ from egon.data import db from egon.data.config import settings -from egon.data.datasets import Dataset +from egon.data.datasets import Dataset, wrapped_partial +from egon.data.metadata import ( + context, + generate_resource_fields_from_sqla_model, + license_ccby, + meta_metadata, + sources, +) import egon.data.config -# will be later imported from another file Base = declarative_base() @@ -57,44 +66,6 @@ class NEP2021ConvPowerPlants(Base): b2040_capacity = Column(Float) -class ScenarioCapacities(Dataset): - """ - Create and fill table with installed generation capacities in Germany - - This dataset creates and fills a table with the installed generation capacities in - Germany in a lower spatial resolution (either per federal state or on national level). - This data is coming from external sources (e.g. German grid developement plan for scenario eGon2035). - The table is in downstream datasets used to define target values for the installed capacities. - - - *Dependencies* - * :py:func:`Setup ` - * :py:class:`PypsaEurSec ` - * :py:class:`Vg250 ` - * :py:class:`DataBundle ` - * :py:class:`ZensusPopulation ` - - - *Resulting tables* - * :py:class:`supply.egon_scenario_capacities ` is created and filled - * :py:class:`supply.egon_nep_2021_conventional_powerplants ` is created and filled - - """ - - #: - name: str = "ScenarioCapacities" - #: - version: str = "0.0.13" - - def __init__(self, dependencies): - super().__init__( - name=self.name, - version=self.version, - dependencies=dependencies, - tasks=(create_table, insert_data_nep, eGon100_capacities), - ) - - def create_table(): """Create input tables for scenario setup @@ -135,6 +106,93 @@ def nuts_mapping(): return nuts_mapping +def insert_capacities_status_quo(scenario: str) -> None: + """Insert capacity of rural heat pumps for status quo + + Returns + ------- + None. + + """ + + targets = egon.data.config.datasets()["scenario_input"]["targets"] + + # Delete rows if already exist + db.execute_sql( + f""" + DELETE FROM + {targets['scenario_capacities']['schema']}. + {targets['scenario_capacities']['table']} + WHERE scenario_name = '{scenario}' + """ + ) + + rural_heat_capacity = { + # Rural heat capacity for 2019 according to NEP 2035, version 2021 + "status2019": 1e6 * 5e-3, + # Rural heat capacity for 2023 according to NEP 2037, version 2023 + # 1.2 Mio. for 2020 + # https://www.netzentwicklungsplan.de/sites/default/files/2023-07/ + # NEP_2037_2045_V2023_2_Entwurf_Teil1_1.pdf#page=25 + # and 3 kW per heat pump + # https://www.netzentwicklungsplan.de/sites/default/files/2022-11/ + # NEP_2035_V2021_2_Entwurf_Teil1.pdf#page=33 + # plus 0.15 Mio. 2021 and 0.24 Mio. in 2022 + # https://www.enercity.de/magazin/unsere-welt/waermepumpen-boom + # plus 0.2 Mio. in H1 2023 -> Assumption 2023: 2 * 0.2 Mio = 0.4 Mio. + "status2023": (1.2 + 0.15 + 0.24 + 0.4) * 1e6 * 3e-3, + }[scenario] + + if settings()["egon-data"]["--dataset-boundary"] != "Everything": + rural_heat_capacity *= population_share() + + db.execute_sql( + f""" + INSERT INTO + {targets['scenario_capacities']['schema']}. + {targets['scenario_capacities']['table']} + (component, carrier, capacity, nuts, scenario_name) + VALUES ( + 'link', + 'residential_rural_heat_pump', + {rural_heat_capacity}, + 'DE', + '{scenario}' + ) + """ + ) + + # Include small storages for scenario2019 + small_storages = { + # MW for Germany + "status2019": 600, + # 1.3 GW in 2020/2021 + # https://www.netzentwicklungsplan.de/sites/default/files/2023-07/ + # NEP_2037_2045_V2023_2_Entwurf_Teil1_1.pdf#page=25 + # Installed quantity 2020: 272,000 + # Installed quantity 2023: 1,197,000 + # https://www.photovoltaik.eu/solarspeicher/ + # bsw-speicherkapazitaet-von-heimspeichern-2023-verdoppelt + "status2023": 1300 * 1197 / 272, + }[scenario] + + db.execute_sql( + f""" + INSERT INTO + {targets['scenario_capacities']['schema']}. + {targets['scenario_capacities']['table']} + (component, carrier, capacity, nuts, scenario_name) + VALUES ( + 'storage_units', + 'battery', + {small_storages}, + 'DE', + '{scenario}' + ) + """ + ) + + def insert_capacities_per_federal_state_nep(): """Inserts installed capacities per federal state accordning to NEP 2035 (version 2021), scenario 2035 C @@ -286,7 +344,7 @@ def insert_capacities_per_federal_state_nep(): # convert GW to MW data.capacity *= 1e3 - insert_data = insert_data.append(data) + insert_data = pd.concat([insert_data, data]) # Get aggregated capacities from nep's power plant list for certain carrier @@ -408,12 +466,16 @@ def map_carrier(): "Braunkohle": "lignite", "Waerme": "others", "Mineraloelprodukte": "oil", + "Mineralölprodukte": "oil", "NichtBiogenerAbfall": "others", + "nicht biogener Abfall": "others", "AndereGase": "gas", + "andere Gase": "gas", "Sonstige_Energietraeger": "others", "Kernenergie": "nuclear", "Pumpspeicher": "pumped_hydro", "Mineralöl-\nProdukte": "oil", + "Biomasse": "biomass", } ) @@ -692,14 +754,17 @@ def eGon100_capacities(): df.index = df.index.str.replace(" ", "_") # Aggregate offshore wind - df = df.append( - pd.DataFrame( - index=["wind_offshore"], - data={ - "p_nom": (df.p_nom["offwind-ac"] + df.p_nom["offwind-dc"]), - "component": df.component["offwind-ac"], - }, - ) + df = pd.concat( + [ + df, + pd.DataFrame( + index=["wind_offshore"], + data={ + "p_nom": (df.p_nom["offwind-ac"] + df.p_nom["offwind-dc"]), + "component": df.component["offwind-ac"], + }, + ), + ] ) df = df.drop(["offwind-ac", "offwind-dc"]) @@ -716,19 +781,22 @@ def eGon100_capacities(): "rural_solar_thermal", ]: if f"residential_{merge_carrier}" in df.index: - df = df.append( - pd.DataFrame( - index=[merge_carrier], - data={ - "p_nom": ( - df.p_nom[f"residential_{merge_carrier}"] - + df.p_nom[f"services_{merge_carrier}"] - ), - "component": df.component[ - f"residential_{merge_carrier}" - ], - }, - ) + df = pd.concat( + [ + df, + pd.DataFrame( + index=[merge_carrier], + data={ + "p_nom": ( + df.p_nom[f"residential_{merge_carrier}"] + + df.p_nom[f"services_{merge_carrier}"] + ), + "component": df.component[ + f"residential_{merge_carrier}" + ], + }, + ), + ] ) df = df.drop( [f"residential_{merge_carrier}", f"services_{merge_carrier}"] @@ -776,3 +844,165 @@ def eGon100_capacities(): if_exists="append", index=False, ) + + +def add_metadata(): + """Add metdata to supply.egon_scenario_capacities + + Returns + ------- + None. + + """ + + # Import column names and datatypes + fields = pd.DataFrame( + generate_resource_fields_from_sqla_model(EgonScenarioCapacities) + ).set_index("name") + + # Set descriptions and units + fields.loc["index", "description"] = "Index" + fields.loc[ + "component", "description" + ] = "Name of representative PyPSA component" + fields.loc["carrier", "description"] = "Name of carrier" + fields.loc["capacity", "description"] = "Installed capacity" + fields.loc["capacity", "unit"] = "MW" + fields.loc[ + "nuts", "description" + ] = "NUTS region, either federal state or Germany" + fields.loc[ + "scenario_name", "description" + ] = "Name of corresponding eGon scenario" + + # Reformat pandas.DataFrame to dict + fields = fields.reset_index().to_dict(orient="records") + + meta = { + "name": "supply.egon_scenario_capacities", + "title": "eGon scenario capacities", + "id": "WILL_BE_SET_AT_PUBLICATION", + "description": ( + "Installed capacities of scenarios used in the eGon project" + ), + "language": ["de-DE"], + "publicationDate": datetime.date.today().isoformat(), + "context": context(), + "spatial": { + "location": None, + "extent": "Germany", + "resolution": None, + }, + "sources": [ + sources()["nep2021"], + sources()["vg250"], + sources()["zensus"], + sources()["egon-data"], + ], + "licenses": [ + license_ccby( + "© Übertragungsnetzbetreiber; " + "© Bundesamt für Kartographie und Geodäsie 2020 (Daten verändert); " + "© Statistische Ämter des Bundes und der Länder 2014; " + "© Jonathan Amme, Clara Büttner, Ilka Cußmann, Julian Endres, Carlos Epia, Stephan Günther, Ulf Müller, Amélia Nadal, Guido Pleßmann, Francesco Witte", + ) + ], + "contributors": [ + { + "title": "Clara Büttner", + "email": "http://github.com/ClaraBuettner", + "date": time.strftime("%Y-%m-%d"), + "object": None, + "comment": "Imported data", + }, + ], + "resources": [ + { + "profile": "tabular-data-resource", + "name": "supply.egon_scenario_capacities", + "path": None, + "format": "PostgreSQL", + "encoding": "UTF-8", + "schema": { + "fields": fields, + "primaryKey": ["index"], + "foreignKeys": [], + }, + "dialect": {"delimiter": None, "decimalSeparator": "."}, + } + ], + "metaMetadata": meta_metadata(), + } + + # Create json dump + meta_json = "'" + json.dumps(meta) + "'" + + # Add metadata as a comment to the table + db.submit_comment( + meta_json, + EgonScenarioCapacities.__table__.schema, + EgonScenarioCapacities.__table__.name, + ) + +tasks = (create_table,) + +scenarios = egon.data.config.settings()["egon-data"]["--scenarios"] + +status_quo = False + +for scenario in scenarios: + if "status" in scenario: + tasks += ( + wrapped_partial( + insert_capacities_status_quo, scenario=scenario, + postfix=f"_{scenario[-2:]}" + ), + ) + status_quo = True + +if status_quo or ("eGon2035" in scenarios): + tasks += (insert_data_nep,) + +if "eGon100RE" in scenarios: + tasks += (eGon100_capacities,) + +tasks += (add_metadata,) + + +class ScenarioCapacities(Dataset): + """ + Create and fill table with installed generation capacities in Germany + + This dataset creates and fills a table with the installed generation capacities in + Germany in a lower spatial resolution (either per federal state or on national level). + This data is coming from external sources (e.g. German grid developement plan for scenario eGon2035). + The table is in downstream datasets used to define target values for the installed capacities. + + + *Dependencies* + * :py:func:`Setup ` + * :py:class:`PypsaEurSec ` + * :py:class:`Vg250 ` + * :py:class:`DataBundle ` + * :py:class:`ZensusPopulation ` + + + *Resulting tables* + * :py:class:`supply.egon_scenario_capacities ` is created and filled + * :py:class:`supply.egon_nep_2021_conventional_powerplants ` is created and filled + + """ + + #: + name: str = "ScenarioCapacities" + #: + version: str = "0.0.13" + + def __init__(self, dependencies): + super().__init__( + name=self.name, + version=self.version, + dependencies=dependencies, + tasks=tasks, + ) + diff --git a/src/egon/data/datasets/scenario_parameters/__init__.py b/src/egon/data/datasets/scenario_parameters/__init__.py index 73ef9196e..7e2ba357e 100755 --- a/src/egon/data/datasets/scenario_parameters/__init__.py +++ b/src/egon/data/datasets/scenario_parameters/__init__.py @@ -1,5 +1,6 @@ """The central module containing all code dealing with scenario table. """ + from pathlib import Path from urllib.request import urlretrieve import shutil @@ -45,6 +46,19 @@ def create_table(): EgonScenario.__table__.create(bind=engine, checkfirst=True) +def get_scenario_year(scenario_name): + """Derives scenarios year from scenario name. Scenario + eGon100RE is an exception as year is not in the name.""" + try: + year = int(scenario_name[-4:]) + except ValueError as e: + if e.args[0] == "invalid literal for int() with base 10: '00RE'": + year = 2050 # eGon100RE + else: + raise ValueError("The names of the scenarios do not end with the year!") + return year + + def insert_scenarios(): """Insert scenarios and their parameters to scenario table @@ -126,6 +140,47 @@ def insert_scenarios(): session.commit() + # Scenario status2019 + status2019 = EgonScenario(name="status2019") + + status2019.description = """ + Status quo ante scenario for 2019 for validation use within the project PoWerD. + """ + status2019.global_parameters = parameters.global_settings(status2019.name) + + status2019.electricity_parameters = parameters.electricity(status2019.name) + + status2019.gas_parameters = parameters.gas(status2019.name) + + status2019.heat_parameters = parameters.heat(status2019.name) + + status2019.mobility_parameters = parameters.mobility(status2019.name) + + session.add(status2019) + + session.commit() + + # Scenario status2023 + status2023 = EgonScenario(name="status2023") + + status2023.description = """ + Status quo ante scenario for 2023. + """ + # TODO status2023 all settings from 2019 are used + status2023.global_parameters = parameters.global_settings(status2023.name) + + status2023.electricity_parameters = parameters.electricity(status2019.name) + + status2023.gas_parameters = parameters.gas(status2019.name) + + status2023.heat_parameters = parameters.heat(status2019.name) + + status2023.mobility_parameters = parameters.mobility(status2023.name) + + session.add(status2023) + + session.commit() + def get_sector_parameters(sector, scenario=None): """Returns parameters for each sector as dictionary. @@ -164,17 +219,17 @@ def get_sector_parameters(sector, scenario=None): else: print(f"Scenario name {scenario} is not valid.") else: - values = ( - pd.DataFrame( - db.select_dataframe( - f""" + values = pd.concat( + [ + pd.DataFrame( + db.select_dataframe( + f""" SELECT {sector}_parameters as val FROM scenario.egon_scenario_parameters WHERE name='eGon2035'""" - ).val[0], - index=["eGon2035"], - ) - .append( + ).val[0], + index=["eGon2035"], + ), pd.DataFrame( db.select_dataframe( f""" @@ -183,9 +238,7 @@ def get_sector_parameters(sector, scenario=None): WHERE name='eGon100RE'""" ).val[0], index=["eGon100RE"], - ) - ) - .append( + ), pd.DataFrame( db.select_dataframe( f""" @@ -194,8 +247,9 @@ def get_sector_parameters(sector, scenario=None): WHERE name='eGon2021'""" ).val[0], index=["eGon2021"], - ) - ) + ), + ], + ignore_index=True, ) return values diff --git a/src/egon/data/datasets/scenario_parameters/parameters.py b/src/egon/data/datasets/scenario_parameters/parameters.py index ed8304c30..3316dc8f1 100755 --- a/src/egon/data/datasets/scenario_parameters/parameters.py +++ b/src/egon/data/datasets/scenario_parameters/parameters.py @@ -7,7 +7,6 @@ def read_csv(year): - source = egon.data.config.datasets()["pypsa-technology-data"]["targets"][ "data_dir" ] @@ -16,7 +15,6 @@ def read_csv(year): def read_costs(df, technology, parameter, value_only=True): - result = df.loc[ (df.technology == technology) & (df.parameter == parameter) ].squeeze() @@ -126,6 +124,61 @@ def global_settings(scenario): "population_year": 2021, } + elif scenario == "status2023": + parameters = { + "weather_year": 2023, + "population_year": 2019, # TODO: check if possible for 2023 + "fuel_costs": { + # TYNDP 2020, data for 2023 (https://2020.entsos-tyndp-scenarios.eu/fuel-commodities-and-carbon-prices/) + "oil": 16.4 * 3.6, # [EUR/MWh] + "gas": 6.1 * 3.6, # [EUR/MWh] + "coal": 3.4 * 3.6, # [EUR/MWh] + "lignite": 1.1 * 3.6, # [EUR/MWh] + "nuclear": 0.47 * 3.6, # [EUR/MWh] + "biomass": read_costs(read_csv(2020), "biomass", "fuel"), + }, + "co2_costs": 83.66, # [EUR/t_CO2], source: + # https://www.iwr.de/news/co2-emissionshandel-deutschland-erzielt-2023-rekordeinnahmen-von-ueber-18-mrd-euro-news38528 + "co2_emissions": { + # Netzentwicklungsplan Strom 2037, Genehmigtr Scenariorahmen, p. 66, table 21 + # https://www.netzentwicklungsplan.de/sites/default/files/2023-01/Szenariorahmen_2037_Genehmigung.pdf + "waste": 0.165, # [t_CO2/MW_th] + "lignite": 0.393, # [t_CO2/MW_th] + "gas": 0.201, # [t_CO2/MW_th] + "nuclear": 0.0, # [t_CO2/MW_th] + "oil": 0.288, # [t_CO2/MW_th] + "coal": 0.337, # [t_CO2/MW_th] + "other_non_renewable": 0.268, # [t_CO2/MW_th] + }, + "interest_rate": 0.05, # [p.u.] + } + + elif scenario == "status2019": + parameters = { + "weather_year": 2019, + "population_year": 2019, + "fuel_costs": { # TYNDP 2020, data for 2020 (https://2020.entsos-tyndp-scenarios.eu/fuel-commodities-and-carbon-prices/) + "oil": 12.9*3.6, # [EUR/MWh] + "gas": 5.6*3.6, # [EUR/MWh] + "coal": 3.0*3.6, # [EUR/MWh] + "lignite": 1.1*3.6, # [EUR/MWh] + "nuclear": 0.47*3.6, # [EUR/MWh] + "biomass": read_costs(read_csv(2020), "biomass", "fuel"), + }, + "co2_costs": 24.7, # [EUR/t_CO2], source: + #https://de.statista.com/statistik/daten/studie/1304069/umfrage/preisentwicklung-von-co2-emissionsrechten-in-eu/ + "co2_emissions": { # Netzentwicklungsplan Strom 2035, Version 2021, 1. Entwurf, p. 40, table 8 + "waste": 0.165, # [t_CO2/MW_th] + "lignite": 0.393, # [t_CO2/MW_th] + "gas": 0.201, # [t_CO2/MW_th] + "nuclear": 0.0, # [t_CO2/MW_th] + "oil": 0.288, # [t_CO2/MW_th] + "coal": 0.335, # [t_CO2/MW_th] + "other_non_renewable": 0.268, # [t_CO2/MW_th] + }, + "interest_rate": 0.05, # [p.u.] + } + else: print(f"Scenario name {scenario} is not valid.") @@ -148,7 +201,6 @@ def electricity(scenario): """ if scenario == "eGon2035": - costs = read_csv(2035) parameters = {"grid_topology": "Status Quo"} @@ -162,12 +214,14 @@ def electricity(scenario): ** 0.5, "standing_loss": 0, "max_hours": 6, + "cyclic_state_of_charge": True, }, "pumped_hydro": { "store": read_costs(costs, "PHS", "efficiency") ** 0.5, "dispatch": read_costs(costs, "PHS", "efficiency") ** 0.5, "standing_loss": 0, "max_hours": 6, + "cyclic_state_of_charge": True, }, } # Warning: Electrical parameters are set in osmTGmod, editing these values will not change the data! @@ -292,27 +346,35 @@ def electricity(scenario): # marginal cost can include fuel, C02 and operation and maintenance costs parameters["marginal_cost"] = { "oil": global_settings(scenario)["fuel_costs"]["oil"] + / read_costs(costs, "oil", "efficiency") + read_costs(costs, "oil", "VOM") + global_settings(scenario)["co2_costs"] - * global_settings(scenario)["co2_emissions"]["oil"], + * global_settings(scenario)["co2_emissions"]["oil"] + / read_costs(costs, "oil", "efficiency"), "other_non_renewable": global_settings(scenario)["fuel_costs"][ "gas" - ] + ] / read_costs(costs, "OCGT", "efficiency") + global_settings(scenario)["co2_costs"] * global_settings(scenario)["co2_emissions"][ "other_non_renewable" - ], + ] / read_costs(costs, "OCGT", "efficiency"), "lignite": global_settings(scenario)["fuel_costs"]["lignite"] + / read_costs(costs, "lignite", "efficiency") + read_costs(costs, "lignite", "VOM") + global_settings(scenario)["co2_costs"] - * global_settings(scenario)["co2_emissions"]["lignite"], + * global_settings(scenario)["co2_emissions"]["lignite"] + / read_costs(costs, "lignite", "efficiency"), "coal": global_settings(scenario)["fuel_costs"]["coal"] + / read_costs(costs, "coal", "efficiency") + read_costs(costs, "coal", "VOM") + global_settings(scenario)["co2_costs"] - * global_settings(scenario)["co2_emissions"]["coal"], + * global_settings(scenario)["co2_emissions"]["coal"] + / read_costs(costs, "coal", "efficiency"), "nuclear": global_settings(scenario)["fuel_costs"]["nuclear"] + / read_costs(costs, "nuclear", "efficiency") + read_costs(costs, "nuclear", "VOM"), "biomass": global_settings(scenario)["fuel_costs"]["biomass"] + / read_costs(costs, "biomass", "efficiency") + read_costs(costs, "biomass CHP", "VOM"), "wind_offshore": read_costs(costs, "offwind", "VOM"), "wind_onshore": read_costs(costs, "onwind", "VOM"), @@ -320,7 +382,6 @@ def electricity(scenario): } elif scenario == "eGon100RE": - costs = read_csv(2050) parameters = {"grid_topology": "Status Quo"} @@ -334,12 +395,14 @@ def electricity(scenario): ** 0.5, "standing_loss": 0, "max_hours": 6, + "cyclic_state_of_charge": True, }, "pumped_hydro": { "store": read_costs(costs, "PHS", "efficiency") ** 0.5, "dispatch": read_costs(costs, "PHS", "efficiency") ** 0.5, "standing_loss": 0, "max_hours": 6, + "cyclic_state_of_charge": True, }, } # Warning: Electrical parameters are set in osmTGmod, editing these values will not change the data! @@ -466,6 +529,191 @@ def electricity(scenario): elif scenario == "eGon2021": parameters = {} + elif (scenario == "status2019") or (scenario == "status2023"): + costs = read_csv(2020) + + parameters = {"grid_topology": "Status Quo"} + # Insert effciencies in p.u. + parameters["efficiency"] = { + "oil": read_costs(costs, "oil", "efficiency"), + "battery": { + "store": read_costs(costs, "battery inverter", "efficiency") + ** 0.5, + "dispatch": read_costs(costs, "battery inverter", "efficiency") + ** 0.5, + "standing_loss": 0, + "max_hours": 6, + "cyclic_state_of_charge": True, + }, + "pumped_hydro": { + "store": read_costs(costs, "PHS", "efficiency") ** 0.5, + "dispatch": read_costs(costs, "PHS", "efficiency") ** 0.5, + "standing_loss": 0, + "max_hours": 6, + "cyclic_state_of_charge": True, + }, + } + # Warning: Electrical parameters are set in osmTGmod, editing these values will not change the data! + parameters["electrical_parameters"] = { + "ac_line_110kV": { + "s_nom": 260, # [MVA] + "R": 0.109, # [Ohm/km] + "L": 1.2, # [mH/km] + }, + "ac_cable_110kV": { + "s_nom": 280, # [MVA] + "R": 0.0177, # [Ohm/km] + "L": 0.3, # [mH/km] + }, + "ac_line_220kV": { + "s_nom": 520, # [MVA] + "R": 0.109, # [Ohm/km] + "L": 1.0, # [mH/km] + }, + "ac_cable_220kV": { + "s_nom": 550, # [MVA] + "R": 0.0176, # [Ohm/km] + "L": 0.3, # [mH/km] + }, + "ac_line_380kV": { + "s_nom": 1790, # [MVA] + "R": 0.028, # [Ohm/km] + "L": 0.8, # [mH/km] + }, + "ac_cable_380kV": { + "s_nom": 925, # [MVA] + "R": 0.0175, # [Ohm/km] + "L": 0.3, # [mH/km] + }, + } + + # Insert overnight investment costs + # Source for eHV grid costs: Netzentwicklungsplan Strom 2035, Version 2021, 2. Entwurf + # Source for HV lines and cables: Dena Verteilnetzstudie 2021, p. 146 + parameters["overnight_cost"] = { + "ac_ehv_overhead_line": 2.5e6 + / ( + 2 + * parameters["electrical_parameters"]["ac_line_380kV"]["s_nom"] + ), # [EUR/km/MW] + "ac_ehv_cable": 11.5e6 + / ( + 2 + * parameters["electrical_parameters"]["ac_cable_380kV"][ + "s_nom" + ] + ), # [EUR/km/MW] + "ac_hv_overhead_line": 0.06e6 + / parameters["electrical_parameters"]["ac_line_110kV"][ + "s_nom" + ], # [EUR/km/MW] + "ac_hv_cable": 0.8e6 + / parameters["electrical_parameters"]["ac_cable_110kV"][ + "s_nom" + ], # [EUR/km/MW] + "dc_overhead_line": 0.5e3, # [EUR/km/MW] + "dc_cable": 3.25e3, # [EUR/km/MW] + "dc_inverter": 0.3e6, # [EUR/MW] + "transformer_380_110": 17.33e3, # [EUR/MVA] + "transformer_380_220": 13.33e3, # [EUR/MVA] + "transformer_220_110": 17.5e3, # [EUR/MVA] + "battery inverter": read_costs( + costs, "battery inverter", "investment" + ), + "battery storage": read_costs( + costs, "battery storage", "investment" + ), + } + + parameters["lifetime"] = { + "ac_ehv_overhead_line": read_costs( + costs, "HVAC overhead", "lifetime" + ), + "ac_ehv_cable": read_costs(costs, "HVAC overhead", "lifetime"), + "ac_hv_overhead_line": read_costs( + costs, "HVAC overhead", "lifetime" + ), + "ac_hv_cable": read_costs(costs, "HVAC overhead", "lifetime"), + "dc_overhead_line": read_costs(costs, "HVDC overhead", "lifetime"), + "dc_cable": read_costs(costs, "HVDC overhead", "lifetime"), + "dc_inverter": read_costs(costs, "HVDC inverter pair", "lifetime"), + "transformer_380_110": read_costs( + costs, "HVAC overhead", "lifetime" + ), + "transformer_380_220": read_costs( + costs, "HVAC overhead", "lifetime" + ), + "transformer_220_110": read_costs( + costs, "HVAC overhead", "lifetime" + ), + "battery inverter": read_costs( + costs, "battery inverter", "lifetime" + ), + "battery storage": read_costs( + costs, "battery storage", "lifetime" + ), + } + # Insert annualized capital costs + # lines in EUR/km/MW/a + # transfermer, inverter, battery in EUR/MW/a + parameters["capital_cost"] = {} + + for comp in parameters["overnight_cost"].keys(): + parameters["capital_cost"][comp] = annualize_capital_costs( + parameters["overnight_cost"][comp], + parameters["lifetime"][comp], + global_settings("status2019")["interest_rate"], + ) + + parameters["capital_cost"]["battery"] = ( + parameters["capital_cost"]["battery inverter"] + + parameters["efficiency"]["battery"]["max_hours"] + * parameters["capital_cost"]["battery storage"] + ) + + parameters["marginal_cost"] = { + "oil": global_settings(scenario)["fuel_costs"]["oil"] + / read_costs(costs, "oil", "efficiency") + + read_costs(costs, "oil", "VOM") + + global_settings(scenario)["co2_costs"] + * global_settings(scenario)["co2_emissions"]["oil"] + / read_costs(costs, "oil", "efficiency"), + "other_non_renewable": global_settings(scenario)["fuel_costs"][ + "gas" + ] / read_costs(costs, "OCGT", "efficiency") + + global_settings(scenario)["co2_costs"] + * global_settings(scenario)["co2_emissions"][ + "other_non_renewable" + ] / read_costs(costs, "OCGT", "efficiency"), + "lignite": global_settings(scenario)["fuel_costs"]["lignite"] + / read_costs(costs, "lignite", "efficiency") + + read_costs(costs, "lignite", "VOM") + + global_settings(scenario)["co2_costs"] + * global_settings(scenario)["co2_emissions"]["lignite"] + / read_costs(costs, "lignite", "efficiency"), + "coal": global_settings(scenario)["fuel_costs"]["coal"] + / read_costs(costs, "coal", "efficiency") + + read_costs(costs, "coal", "VOM") + + global_settings(scenario)["co2_costs"] + * global_settings(scenario)["co2_emissions"]["coal"] + / read_costs(costs, "coal", "efficiency"), + "OCGT": global_settings(scenario)["fuel_costs"]["gas"] + / read_costs(costs, "OCGT", "efficiency") + + read_costs(costs, "OCGT", "VOM") + + global_settings(scenario)["co2_costs"] + * global_settings(scenario)["co2_emissions"]["gas"] + / read_costs(costs, "OCGT", "efficiency"), + "nuclear": global_settings(scenario)["fuel_costs"]["nuclear"] + / read_costs(costs, "nuclear", "efficiency") + + read_costs(costs, "nuclear", "VOM"), + "biomass": global_settings(scenario)["fuel_costs"]["biomass"] + / read_costs(costs, "biomass CHP", "efficiency") + + read_costs(costs, "biomass CHP", "VOM"), + "wind_offshore": read_costs(costs, "offwind", "VOM"), + "wind_onshore": read_costs(costs, "onwind", "VOM"), + "solar": read_costs(costs, "solar", "VOM"), + } + else: print(f"Scenario name {scenario} is not valid.") @@ -488,7 +736,6 @@ def gas(scenario): """ if scenario == "eGon2035": - costs = read_csv(2035) parameters = { @@ -564,7 +811,6 @@ def gas(scenario): } elif scenario == "eGon100RE": - costs = read_csv(2050) interest_rate = 0.07 # [p.u.] @@ -642,14 +888,12 @@ def gas(scenario): parameters["capital_cost"] = {} for comp in parameters["overnight_cost"].keys(): - parameters["capital_cost"][comp] = ( - annualize_capital_costs( - parameters["overnight_cost"][comp], - parameters["lifetime"][comp], - interest_rate, - ) - + parameters["overnight_cost"][comp] - * (parameters["FOM"][comp] / 100) + parameters["capital_cost"][comp] = annualize_capital_costs( + parameters["overnight_cost"][comp], + parameters["lifetime"][comp], + interest_rate, + ) + parameters["overnight_cost"][comp] * ( + parameters["FOM"][comp] / 100 ) for comp in ["H2_to_power", "H2_to_CH4"]: @@ -672,6 +916,25 @@ def gas(scenario): elif scenario == "eGon2021": parameters = {} + elif scenario == "status2019": + costs = read_csv(2020) + parameters = { + "main_gas_carrier": "CH4", + } + + parameters["marginal_cost"] = { + "CH4": global_settings(scenario)["fuel_costs"]["gas"] + + global_settings(scenario)["co2_costs"] + * global_settings(scenario)["co2_emissions"]["gas"], + "OCGT": read_costs(costs, "OCGT", "VOM"), + "biogas": global_settings(scenario)["fuel_costs"]["gas"], + "chp_gas": read_costs(costs, "central gas CHP", "VOM"), + } + # Insert effciencies in p.u. + parameters["efficiency"] = { + "OCGT": read_costs(costs, "OCGT", "efficiency"), + } + else: print(f"Scenario name {scenario} is not valid.") @@ -756,6 +1019,38 @@ def mobility(scenario): elif scenario == "eGon2021": parameters = {} + elif scenario == "status2019": + parameters = { + "motorized_individual_travel": { + "status2019": { + "ev_count": 200000, + "bev_mini_share": 0.1589, + "bev_medium_share": 0.3533, + "bev_luxury_share": 0.1053, + "phev_mini_share": 0.0984, + "phev_medium_share": 0.2189, + "phev_luxury_share": 0.0652, + "model_parameters": {}, + } + } + } + + elif scenario == "status2023": + parameters = { + "motorized_individual_travel": { + "status2023": { + "ev_count": 2577664, + "bev_mini_share": 0.1535, + "bev_medium_share": 0.3412, + "bev_luxury_share": 0.1017, + "phev_mini_share": 0.1038, + "phev_medium_share": 0.2310, + "phev_luxury_share": 0.0688, + "model_parameters": {}, + } + } + } + else: print(f"Scenario name {scenario} is not valid.") parameters = dict() @@ -779,7 +1074,6 @@ def heat(scenario): """ if scenario == "eGon2035": - costs = read_csv(2035) parameters = { @@ -869,6 +1163,81 @@ def heat(scenario): elif scenario == "eGon2021": parameters = {} + elif scenario == "status2019": + parameters = { + "DE_demand_residential_TJ": 1658400 + + 383300, # [TJ], space heating + hot water, source: AG Energiebilanzen 2019 (https://ag-energiebilanzen.de/wp-content/uploads/2020/10/ageb_20v_v1.pdf) + "DE_demand_service_TJ": 567300 + + 71500, # [TJ], space heating + hot water, source: AG Energiebilanzen 2019 (https://ag-energiebilanzen.de/wp-content/uploads/2020/10/ageb_20v_v1.pdf) + "DE_district_heating_share": (189760 + 38248) + / ( + 1658400 + 383300 + 567300 + 71500 + ), # [TJ], source: AG Energiebilanzen 2019 (https://ag-energiebilanzen.de/wp-content/uploads/2021/11/bilanz19d.xlsx) + } + + costs = read_csv(2020) + + # Insert marginal_costs in EUR/MWh + # marginal cost can include fuel, C02 and operation and maintenance costs + parameters["marginal_cost"] = { + "central_heat_pump": read_costs( + costs, "central air-sourced heat pump", "VOM" + ), + "central_gas_chp": read_costs(costs, "central gas CHP", "VOM"), + "central_gas_boiler": read_costs( + costs, "central gas boiler", "VOM" + ), + "central_resistive_heater": read_costs( + costs, "central resistive heater", "VOM" + ), + "rural_heat_pump": 0, # Danish Energy Agency, Technology Data for Individual Heating Plants + } + + # Insert efficiency in p.u. + parameters["efficiency"] = { + "central_gas_boiler": read_costs( + costs, "central gas boiler", "efficiency" + ), + } + + # elif scenario == "status2023": + # parameters = { + # # source: AG Energiebilanzen 2022 https://ag-energiebilanzen.de/wp-content/uploads/2023/01/AGEB_22p2_rev-1.pdf + # "DE_demand_residential_TJ": 1754.2 * 1e3 + # + 407.5 * 1e3, # [TJ], Endenergieverbrauch Haushalte 2.1 Raumwärme + Warmwasser + # "DE_demand_service_TJ": 668.4 * 1e3 + # + 44.3 * 1e3 , # [TJ], Endenergieverbrauch GHD 3.1 Raumwärme + Warmwasser + # "DE_district_heating_share": (189760 + 38248) + # / ( + # 1658400 + 383300 + 567300 + 71500 + # ), # [TJ], source: AG Energiebilanzen 2019 (https://ag-energiebilanzen.de/wp-content/uploads/2021/11/bilanz19d.xlsx) + # } # TODO status2023 needs update + # + # costs = read_csv(2020) + # + # # Insert marginal_costs in EUR/MWh + # # marginal cost can include fuel, C02 and operation and maintenance costs + # parameters["marginal_cost"] = { + # "central_heat_pump": read_costs( + # costs, "central air-sourced heat pump", "VOM" + # ), + # "central_gas_chp": read_costs(costs, "central gas CHP", "VOM"), + # "central_gas_boiler": read_costs( + # costs, "central gas boiler", "VOM" + # ), + # "central_resistive_heater": read_costs( + # costs, "central resistive heater", "VOM" + # ), + # "rural_heat_pump": 0, # Danish Energy Agency, Technology Data for Individual Heating Plants + # } + # + # # Insert efficiency in p.u. + # parameters["efficiency"] = { + # "central_gas_boiler": read_costs( + # costs, "central gas boiler", "efficiency" + # ), + # } + else: print(f"Scenario name {scenario} is not valid.") diff --git a/src/egon/data/datasets/society_prognosis.py b/src/egon/data/datasets/society_prognosis.py index 1e17f66ab..48c70b6ab 100755 --- a/src/egon/data/datasets/society_prognosis.py +++ b/src/egon/data/datasets/society_prognosis.py @@ -89,7 +89,7 @@ def zensus_population(): zensus.groupby(zensus.nuts3) .population.apply(lambda grp: grp / grp.sum()) .fillna(0) - ) + ).values db.execute_sql( f"""DELETE FROM {cfg['target']['population_prognosis']['schema']}. diff --git a/src/egon/data/datasets/storages/__init__.py b/src/egon/data/datasets/storages/__init__.py index f503d4e85..9d86aa0f1 100755 --- a/src/egon/data/datasets/storages/__init__.py +++ b/src/egon/data/datasets/storages/__init__.py @@ -1,5 +1,6 @@ """The central module containing all code dealing with power plant data. """ + from pathlib import Path from geoalchemy2 import Geometry @@ -12,8 +13,12 @@ from egon.data import config, db from egon.data.datasets import Dataset -from egon.data.datasets.mastr import WORKING_DIR_MASTR_OLD -from egon.data.datasets.power_plants import assign_voltage_level +from egon.data.datasets.mastr import ( + WORKING_DIR_MASTR_NEW, + WORKING_DIR_MASTR_OLD, +) +from egon.data.datasets.mv_grid_districts import Vg250GemClean +from egon.data.datasets.power_plants import assign_bus_id, assign_voltage_level from egon.data.datasets.storages.home_batteries import ( allocate_home_batteries_to_buildings, ) @@ -24,6 +29,7 @@ select_mastr_pumped_hydro, select_nep_pumped_hydro, ) +from egon.data.db import session_scope Base = declarative_base() @@ -46,14 +52,13 @@ class Storages(Dataset): def __init__(self, dependencies): super().__init__( name="Storages", - version="0.0.4", + version="0.0.8", dependencies=dependencies, tasks=( create_tables, - allocate_pumped_hydro_eGon2035, - allocate_pumped_hydro_eGon100RE, + allocate_pumped_hydro_scn, allocate_pv_home_batteries_to_grids, - allocate_home_batteries_to_buildings, + # allocate_home_batteries_to_buildings, ), ) @@ -77,9 +82,9 @@ def create_tables(): EgonStorages.__table__.create(bind=engine, checkfirst=True) -def allocate_pumped_hydro_eGon2035(export=True): - """Allocates pumped_hydro plants for eGon2035 scenario and either exports - results to data base or returns as a dataframe +def allocate_pumped_hydro(scn, export=True): + """Allocates pumped_hydro plants for eGon2035 and scenario2019 scenarios + and either exports results to data base or returns as a dataframe Parameters ---------- @@ -91,21 +96,21 @@ def allocate_pumped_hydro_eGon2035(export=True): Returns ------- power_plants : pandas.DataFrame - List of pumped hydro plants in 'eGon2035' scenario + List of pumped hydro plants in 'eGon2035' and 'scenario2019' scenarios """ carrier = "pumped_hydro" cfg = config.datasets()["power_plants"] - nep = select_nep_pumped_hydro() + nep = select_nep_pumped_hydro(scn=scn) mastr = select_mastr_pumped_hydro() # Assign voltage level to MaStR mastr["voltage_level"] = assign_voltage_level( mastr.rename({"el_capacity": "Nettonennleistung"}, axis=1), cfg, - WORKING_DIR_MASTR_OLD + WORKING_DIR_MASTR_OLD, ) # Initalize DataFrame for matching power plants @@ -124,7 +129,12 @@ def allocate_pumped_hydro_eGon2035(export=True): # Match pumped_hydro units from NEP list # using PLZ and capacity matched, mastr, nep = match_storage_units( - nep, mastr, matched, buffer_capacity=0.1, consider_carrier=False + nep, + mastr, + matched, + buffer_capacity=0.1, + consider_carrier=False, + scn=scn, ) # Match plants from NEP list using plz, @@ -136,6 +146,7 @@ def allocate_pumped_hydro_eGon2035(export=True): consider_location="plz", consider_carrier=False, consider_capacity=False, + scn=scn, ) # Match plants from NEP list using city, @@ -147,6 +158,7 @@ def allocate_pumped_hydro_eGon2035(export=True): consider_location="city", consider_carrier=False, consider_capacity=False, + scn=scn, ) # Match remaining plants from NEP using the federal state @@ -157,6 +169,7 @@ def allocate_pumped_hydro_eGon2035(export=True): buffer_capacity=0.1, consider_location="federal_state", consider_carrier=False, + scn=scn, ) # Match remaining plants from NEP using the federal state @@ -167,27 +180,30 @@ def allocate_pumped_hydro_eGon2035(export=True): buffer_capacity=0.7, consider_location="federal_state", consider_carrier=False, + scn=scn, ) print(f"{matched.el_capacity.sum()} MW of {carrier} matched") - print(f"{nep.c2035_capacity.sum()} MW of {carrier} not matched") - - if nep.c2035_capacity.sum() > 0: + print(f"{nep.elec_capacity.sum()} MW of {carrier} not matched") + if nep.elec_capacity.sum() > 0: # Get location using geolocator and city information located, unmatched = get_location(nep) # Bring both dataframes together - matched = matched.append( - located[ - [ - "carrier", - "el_capacity", - "scenario", - "geometry", - "source", - "MaStRNummer", - ] + matched = pd.concat( + [ + matched, + located[ + [ + "carrier", + "el_capacity", + "scenario", + "geometry", + "source", + "MaStRNummer", + ] + ], ], ignore_index=True, ) @@ -231,9 +247,9 @@ def allocate_pumped_hydro_eGon2035(export=True): # Delete existing units in the target table db.execute_sql( - f""" DELETE FROM {cfg ['target']['schema']}.{cfg ['target']['table']} + f""" DELETE FROM {cfg ['sources']['storages']} WHERE carrier IN ('pumped_hydro') - AND scenario='eGon2035';""" + AND scenario='{scn}';""" ) # If export = True export pumped_hydro plants to data base @@ -241,24 +257,238 @@ def allocate_pumped_hydro_eGon2035(export=True): if export: # Insert into target table session = sessionmaker(bind=db.engine())() - for i, row in power_plants.iterrows(): - entry = EgonStorages( - sources={"el_capacity": row.source}, - source_id={"MastrNummer": row.MaStRNummer}, - carrier=row.carrier, - el_capacity=row.el_capacity, - voltage_level=row.voltage_level, - bus_id=row.bus_id, - scenario=row.scenario, - geom=f"SRID=4326;POINT({row.geometry.x} {row.geometry.y})", - ) - session.add(entry) - session.commit() + with session_scope() as session: + for i, row in power_plants.iterrows(): + entry = EgonStorages( + sources={"el_capacity": row.source}, + source_id={"MastrNummer": row.MaStRNummer}, + carrier=row.carrier, + el_capacity=row.el_capacity, + voltage_level=row.voltage_level, + bus_id=row.bus_id, + scenario=row.scenario, + geom=f"SRID=4326;POINT({row.geometry.x} {row.geometry.y})", + ) + session.add(entry) + session.commit() else: return power_plants +def allocate_pumped_hydro_sq(scn_name): + """ + Allocate pumped hydro by mastr data only. Capacities outside + germany are assigned to foreign buses. Mastr dump 2024 is used. + No filter for commissioning is applied. + Parameters + ---------- + scn_name + + Returns + ------- + + """ + sources = config.datasets()["power_plants"]["sources"] + + # Read-in data from MaStR + mastr_ph = pd.read_csv( + WORKING_DIR_MASTR_NEW / sources["mastr_storage"], + delimiter=",", + usecols=[ + "Nettonennleistung", + "EinheitMastrNummer", + "Kraftwerksnummer", + "Technologie", + "Postleitzahl", + "Laengengrad", + "Breitengrad", + "EinheitBetriebsstatus", + "LokationMastrNummer", + "Ort", + "Bundesland", + ], + dtype={"Postleitzahl": str}, + ) + + if config.settings()["egon-data"]["--dataset-boundary"] == "Schleswig-Holstein": + # Filter for Schleswig-Holstein + mastr_ph = mastr_ph.loc[mastr_ph.Bundesland == "SchleswigHolstein"] + + # Rename columns + mastr_ph = mastr_ph.rename( + columns={ + "Kraftwerksnummer": "bnetza_id", + "Technologie": "carrier", + "Postleitzahl": "plz", + "Ort": "city", + "Bundesland": "federal_state", + "Nettonennleistung": "el_capacity", + } + ) + + # Select only pumped hydro units + mastr_ph = mastr_ph.loc[mastr_ph.carrier == "Pumpspeicher"] + + # Select only pumped hydro units which are in operation + mastr_ph = mastr_ph.loc[mastr_ph.EinheitBetriebsstatus == "InBetrieb"] + + # Calculate power in MW + mastr_ph.loc[:, "el_capacity"] *= 1e-3 + + # Create geodataframe from long, lat + mastr_ph = gpd.GeoDataFrame( + mastr_ph, + geometry=gpd.points_from_xy( + mastr_ph["Laengengrad"], mastr_ph["Breitengrad"] + ), + crs="4326", + ) + + # Identify pp without geocord + mastr_ph_nogeo = mastr_ph.loc[mastr_ph["Laengengrad"].isna()] + + # Remove all PP without geocord (PP<= 30kW) + mastr_ph = mastr_ph.dropna(subset="Laengengrad") + + # Get geometry of villages/cities with same name of pp with missing geocord + with session_scope() as session: + query = session.query( + Vg250GemClean.gen, Vg250GemClean.geometry + ).filter(Vg250GemClean.gen.in_(mastr_ph_nogeo.loc[:, "city"].unique())) + df_cities = gpd.read_postgis( + query.statement, + query.session.bind, + geom_col="geometry", + crs="4326", + ) + + # Just take the first entry, inaccuracy is negligible as centroid is taken afterwards + df_cities = df_cities.drop_duplicates("gen", keep="first") + + # Use the centroid instead of polygon of region + df_cities.loc[:, "geometry"] = df_cities["geometry"].centroid + + # Add centroid geometry to pp without geometry + mastr_ph_nogeo = pd.merge( + left=df_cities, + right=mastr_ph_nogeo, + right_on="city", + left_on="gen", + how="inner", + ).drop("gen", axis=1) + + mastr_ph = pd.concat([mastr_ph, mastr_ph_nogeo], axis=0) + + # aggregate capacity per location + agg_cap = mastr_ph.groupby("geometry")["el_capacity"].sum() + + # list mastr number by location + agg_mastr = mastr_ph.groupby("geometry")["EinheitMastrNummer"].apply(list) + + # remove duplicates by location and keep only first + mastr_ph = mastr_ph.drop_duplicates(subset="geometry", keep="first").drop( + ["el_capacity", "EinheitMastrNummer"], axis=1 + ) + + # Add aggregated capacity by location + mastr_ph = pd.merge( + left=mastr_ph, right=agg_cap, left_on="geometry", right_on="geometry" + ) + + # Add list of mastr nr by location + mastr_ph = pd.merge( + left=mastr_ph, right=agg_mastr, left_on="geometry", right_on="geometry" + ) + + # Drop small pp <= 03 kW + mastr_ph = mastr_ph.loc[mastr_ph["el_capacity"] > 30] + + # Apply voltage level by capacity + mastr_ph = apply_voltage_level_thresholds(mastr_ph) + mastr_ph["voltage_level"] = mastr_ph["voltage_level"].astype(int) + + # Capacity located outside germany -> will be assigned to foreign buses + mastr_ph_foreign = mastr_ph.loc[mastr_ph["federal_state"].isna()] + + if not mastr_ph_foreign.empty: + # Get foreign buses + sql = f""" + SELECT * FROM grid.egon_etrago_bus + WHERE scn_name = '{scn_name}' + and country != 'DE' + """ + df_foreign_buses = db.select_geodataframe( + sql, geom_col="geom", epsg="4326" + ) + + # Assign closest foreign bus at voltage level to foreign pp + nearest_neighbors = [] + for vl, v_nom in {1: 380, 2: 220, 3: 110}.items(): + ph = mastr_ph_foreign.loc[mastr_ph_foreign["voltage_level"] == vl] + if ph.empty: + continue + bus = df_foreign_buses.loc[ + df_foreign_buses["v_nom"] == v_nom, + ["v_nom", "country", "bus_id", "geom"], + ] + results = gpd.sjoin_nearest( + left_df=ph, right_df=bus, how="left", distance_col="distance" + ) + nearest_neighbors.append(results) + mastr_ph_foreign = pd.concat(nearest_neighbors) + + # Keep only capacities within germany + mastr_ph = mastr_ph.dropna(subset="federal_state") + + # Assign buses within germany + mastr_ph = assign_bus_id(mastr_ph, cfg=config.datasets()["power_plants"]) + mastr_ph["bus_id"] = mastr_ph["bus_id"].astype(int) + + if not mastr_ph_foreign.empty: + # Merge foreign pp + mastr_ph = pd.concat([mastr_ph, mastr_ph_foreign]) + + # Reduce to necessary columns + mastr_ph = mastr_ph[ + [ + "el_capacity", + "voltage_level", + "bus_id", + "geometry", + "EinheitMastrNummer", + ] + ] + + # Rename and format columns + mastr_ph["carrier"] = "pumped_hydro" + mastr_ph = mastr_ph.rename( + columns={"EinheitMastrNummer": "source_id", "geometry": "geom"} + ) + mastr_ph["source_id"] = mastr_ph["source_id"].apply( + lambda x: {"MastrNummer": ", ".join(x)} + ) + mastr_ph = mastr_ph.set_geometry("geom") + mastr_ph["geom"] = mastr_ph["geom"].apply(lambda x: x.wkb_hex) + mastr_ph["scenario"] = "status2023" + mastr_ph["sources"] = [ + {"el_capacity": "MaStR aggregated by location"} + ] * mastr_ph.shape[0] + + # Delete existing units in the target table + db.execute_sql( + f""" DELETE FROM supply.egon_storages + WHERE carrier = 'pumped_hydro' + AND scenario= '{scn_name}';""" + ) + + with db.session_scope() as session: + session.bulk_insert_mappings( + EgonStorages, + mastr_ph.to_dict(orient="records"), + ) + + def allocate_pumped_hydro_eGon100RE(): """Allocates pumped_hydro plants for eGon100RE scenario based on a prox-to-now method applied on allocated pumped-hydro plants in the eGon2035 @@ -301,7 +531,8 @@ def allocate_pumped_hydro_eGon100RE(): # Get allocation of pumped_hydro plants in eGon2035 scenario as the # reference for the distribution in eGon100RE scenario - allocation = allocate_pumped_hydro_eGon2035(export=False) + allocation = allocate_pumped_hydro(scn="status2019", export=False) + # TODO status2023 leave same as status2019 scaling_factor = capacity_phes / allocation.el_capacity.sum() @@ -345,7 +576,6 @@ def home_batteries_per_scenario(scenario): dataset = config.settings()["egon-data"]["--dataset-boundary"] if scenario == "eGon2035": - target_file = ( Path(".") / "data_bundle_egon_data" @@ -359,8 +589,8 @@ def home_batteries_per_scenario(scenario): index_col="Unnamed: 0", ) - # Select target value in MW - target = capacities_nep.Summe["PV-Batteriespeicher"]*1000 + # Select target value in MW + target = capacities_nep.Summe["PV-Batteriespeicher"] * 1000 else: target = db.select_dataframe( @@ -394,7 +624,7 @@ def home_batteries_per_scenario(scenario): battery["carrier"] = "home_battery" battery["scenario"] = scenario - if scenario == "eGon2035": + if (scenario == "eGon2035") | ("status" in scenario): source = "NEP" else: @@ -420,6 +650,15 @@ def home_batteries_per_scenario(scenario): def allocate_pv_home_batteries_to_grids(): - - home_batteries_per_scenario("eGon2035") - home_batteries_per_scenario("eGon100RE") + for scn in config.settings()["egon-data"]["--scenarios"]: + home_batteries_per_scenario(scn) + + +def allocate_pumped_hydro_scn(): + for scn in config.settings()["egon-data"]["--scenarios"]: + if scn == "eGon2035": + allocate_pumped_hydro(scn="eGon2035") + elif scn == "eGon100RE": + allocate_pumped_hydro_eGon100RE() + elif "status" in scn: + allocate_pumped_hydro_sq(scn_name=scn) diff --git a/src/egon/data/datasets/storages/home_batteries.py b/src/egon/data/datasets/storages/home_batteries.py index 40d19c1ea..75448e67e 100644 --- a/src/egon/data/datasets/storages/home_batteries.py +++ b/src/egon/data/datasets/storages/home_batteries.py @@ -32,14 +32,29 @@ The selection of buildings is done randomly until a result is reached which is close to achieving the sizing specification. """ +import datetime +import json + from loguru import logger from numpy.random import RandomState +from omi.dialects import get_dialect from sqlalchemy import Column, Float, Integer, String from sqlalchemy.ext.declarative import declarative_base import numpy as np import pandas as pd from egon.data import config, db +from egon.data.datasets.scenario_parameters import get_sector_parameters +from egon.data.metadata import ( + context, + contributors, + generate_resource_fields_from_db_table, + license_dedl, + license_odbl, + meta_metadata, + meta_metadata, + sources, +) Base = declarative_base() @@ -73,11 +88,10 @@ def allocate_home_batteries_to_buildings(): """ # get constants constants = config.datasets()["home_batteries"]["constants"] - scenarios = constants["scenarios"] + scenarios = config.settings()["egon-data"]["--scenarios"] cbat_ppv_ratio = constants["cbat_ppv_ratio"] rtol = constants["rtol"] max_it = constants["max_it"] - cbat_pbat_ratio = get_cbat_pbat_ratio() sources = config.datasets()["home_batteries"]["sources"] @@ -92,7 +106,10 @@ def allocate_home_batteries_to_buildings(): WHERE carrier = 'home_battery' AND scenario = '{scenario}'; """ - + cbat_pbat_ratio = get_sector_parameters( + "electricity", scenario + )["efficiency"]["battery"]["max_hours"] + home_batteries_df = db.select_dataframe(sql) home_batteries_df = home_batteries_df.assign( @@ -173,6 +190,8 @@ def allocate_home_batteries_to_buildings(): create_table(pd.concat(df_list, ignore_index=True)) + add_metadata() + class EgonHomeBatteries(Base): targets = config.datasets()["home_batteries"]["targets"] @@ -188,6 +207,142 @@ class EgonHomeBatteries(Base): capacity = Column(Float) +def add_metadata(): + """ + Add metadata to table supply.egon_home_batteries + """ + targets = config.datasets()["home_batteries"]["targets"] + deposit_id_mastr = config.datasets()["mastr_new"]["deposit_id"] + deposit_id_data_bundle = config.datasets()["data-bundle"]["sources"][ + "zenodo" + ]["deposit_id"] + + contris = contributors(["kh", "kh"]) + + contris[0]["date"] = "2023-03-15" + + contris[0]["object"] = "metadata" + contris[1]["object"] = "dataset" + + contris[0]["comment"] = "Add metadata to dataset." + contris[1]["comment"] = "Add workflow to generate dataset." + + meta = { + "name": ( + f"{targets['home_batteries']['schema']}." + f"{targets['home_batteries']['table']}" + ), + "title": "eGon Home Batteries", + "id": "WILL_BE_SET_AT_PUBLICATION", + "description": "Home storage systems allocated to buildings", + "language": "en-US", + "keywords": ["battery", "batteries", "home", "storage", "building"], + "publicationDate": datetime.date.today().isoformat(), + "context": context(), + "spatial": { + "location": "none", + "extent": "Germany", + "resolution": "building", + }, + "temporal": { + "referenceDate": "2021-12-31", + "timeseries": {}, + }, + "sources": [ + { + "title": "Data bundle for egon-data", + "description": ( + "Data bundle for egon-data: A transparent and " + "reproducible data processing pipeline for energy " + "system modeling" + ), + "path": ( + "https://zenodo.org/record/" + f"{deposit_id_data_bundle}#.Y_dWM4CZMVM" + ), + "licenses": [license_dedl(attribution="© Cußmann, Ilka")], + }, + { + "title": ("open-MaStR power unit registry for eGo^n project"), + "description": ( + "Data from Marktstammdatenregister (MaStR) data using " + "the data dump from 2022-11-17 for eGon-data." + ), + "path": ( + f"https://zenodo.org/record/{deposit_id_mastr}" + ), + "licenses": [license_dedl(attribution="© Amme, Jonathan")], + }, + sources()["openstreetmap"], + sources()["era5"], + sources()["vg250"], + sources()["egon-data"], + sources()["nep2021"], + sources()["mastr"], + sources()["technology-data"], + ], + "licenses": [license_odbl("© eGon development team")], + "contributors": contris, + "resources": [ + { + "profile": "tabular-data-resource", + "name": ( + f"{targets['home_batteries']['schema']}." + f"{targets['home_batteries']['table']}" + ), + "path": "None", + "format": "PostgreSQL", + "encoding": "UTF-8", + "schema": { + "fields": generate_resource_fields_from_db_table( + targets["home_batteries"]["schema"], + targets["home_batteries"]["table"], + ), + "primaryKey": "index", + }, + "dialect": {"delimiter": "", "decimalSeparator": ""}, + } + ], + "review": {"path": "", "badge": ""}, + "metaMetadata": meta_metadata(), + "_comment": { + "metadata": ( + "Metadata documentation and explanation (https://github.com/" + "OpenEnergyPlatform/oemetadata/blob/master/metadata/v141/" + "metadata_key_description.md)" + ), + "dates": ( + "Dates and time must follow the ISO8601 including time zone " + "(YYYY-MM-DD or YYYY-MM-DDThh:mm:ss±hh)" + ), + "units": "Use a space between numbers and units (100 m)", + "languages": ( + "Languages must follow the IETF (BCP47) format (en-GB, en-US, " + "de-DE)" + ), + "licenses": ( + "License name must follow the SPDX License List " + "(https://spdx.org/licenses/)" + ), + "review": ( + "Following the OEP Data Review (https://github.com/" + "OpenEnergyPlatform/data-preprocessing/wiki)" + ), + "none": "If not applicable use (none)", + }, + } + + dialect = get_dialect(meta_metadata())() + + meta = dialect.compile_and_render(dialect.parse(json.dumps(meta))) + + db.submit_comment( + f"'{json.dumps(meta)}'", + targets["home_batteries"]["schema"], + targets["home_batteries"]["table"], + ) + + def create_table(df): """Create mapping table home battery <-> building id""" engine = db.engine() diff --git a/src/egon/data/datasets/storages/pumped_hydro.py b/src/egon/data/datasets/storages/pumped_hydro.py index fa076cc74..c9f2bace3 100755 --- a/src/egon/data/datasets/storages/pumped_hydro.py +++ b/src/egon/data/datasets/storages/pumped_hydro.py @@ -11,7 +11,7 @@ from egon.data import config, db from egon.data.datasets.chp.match_nep import match_nep_chp from egon.data.datasets.chp.small_chp import assign_use_case -from egon.data.datasets.mastr import WORKING_DIR_MASTR_OLD +from egon.data.datasets.mastr import WORKING_DIR_MASTR_NEW from egon.data.datasets.power_plants import ( assign_bus_id, assign_voltage_level, @@ -21,7 +21,7 @@ import egon.data.config -def select_nep_pumped_hydro(): +def select_nep_pumped_hydro(scn): """Select pumped hydro plants from NEP power plants list @@ -34,17 +34,39 @@ def select_nep_pumped_hydro(): carrier = "pumped_hydro" - # Select plants with geolocation from list of conventional power plants - nep_ph = db.select_dataframe( - f""" - SELECT bnetza_id, name, carrier, postcode, capacity, city, - federal_state, c2035_capacity - FROM {cfg['sources']['nep_conv']} - WHERE carrier = '{carrier}' - AND c2035_capacity > 0 - AND postcode != 'None'; - """ - ) + if scn == "eGon2035": + # Select plants with geolocation from list of conventional power plants + nep_ph = db.select_dataframe( + f""" + SELECT bnetza_id, name, carrier, postcode, capacity, city, + federal_state, c2035_capacity + FROM {cfg['sources']['nep_conv']} + WHERE carrier = '{carrier}' + AND c2035_capacity > 0 + AND postcode != 'None'; + """ + ) + nep_ph.rename( + columns={"c2035_capacity": "elec_capacity"}, inplace=True + ) + elif "status" in scn: + # Select plants with geolocation from list of conventional power plants + year = int(scn[-4:]) + + nep_ph = db.select_dataframe( + f""" + SELECT bnetza_id, name, carrier, postcode, capacity, city, + federal_state + FROM {cfg['sources']['nep_conv']} + WHERE carrier = '{carrier}' + AND capacity > 0 + AND postcode != 'None' + AND commissioned < '{year+1}'; + """ + ) + nep_ph["elec_capacity"] = nep_ph["capacity"] + else: + raise SystemExit(f"{scn} not recognised") # Removing plants out of Germany nep_ph["postcode"] = nep_ph["postcode"].astype(str) @@ -71,7 +93,7 @@ def select_mastr_pumped_hydro(): # Read-in data from MaStR mastr_ph = pd.read_csv( - WORKING_DIR_MASTR_OLD / sources["mastr_storage"], + WORKING_DIR_MASTR_NEW / sources["mastr_storage"], delimiter=",", usecols=[ "Nettonennleistung", @@ -115,6 +137,22 @@ def select_mastr_pumped_hydro(): ), ) + mastr_ph = mastr_ph.set_crs(4326) + + # drop hydropower without federal state + # Obervermunterwerk II in Austria + mastr_ph = mastr_ph[~(mastr_ph["federal_state"].isnull())] + + if ( + config.settings()["egon-data"]["--dataset-boundary"] + == "Schleswig-Holstein" + ): + # Drop hydropower outside the test mode area + mastr_ph = filter_mastr_geometry(mastr_ph, federal_state="Schleswig-Holstein") + else: + # Drop hydropower outside of germany + mastr_ph = filter_mastr_geometry(mastr_ph, federal_state=None) + # Drop rows without post code and update datatype of postcode mastr_ph = mastr_ph[~mastr_ph["plz"].isnull()] mastr_ph["plz"] = mastr_ph["plz"].astype(int) @@ -122,12 +160,6 @@ def select_mastr_pumped_hydro(): # Calculate power in MW mastr_ph.loc[:, "el_capacity"] *= 1e-3 - mastr_ph = mastr_ph.set_crs(4326) - - mastr_ph = mastr_ph[~(mastr_ph["federal_state"].isnull())] - - # Drop CHP outside of Germany/ outside the test mode area - mastr_ph = filter_mastr_geometry(mastr_ph, federal_state=None) return mastr_ph @@ -140,6 +172,7 @@ def match_storage_units( consider_location="plz", consider_carrier=True, consider_capacity=True, + scn="eGon2035", ): """Match storage_units (in this case only pumped hydro) from MaStR to list of power plants from NEP @@ -154,6 +187,8 @@ def match_storage_units( Already matched storage_units buffer_capacity : float, optional Maximum difference in capacity in p.u. The default is 0.1. + scn : string, optional + Scenario name Returns ------- @@ -192,7 +227,6 @@ def match_storage_units( for index, row in nep[ (nep["carrier"] == carrier) & (nep["postcode"] != "None") ].iterrows(): - # Select plants from MaStR that match carrier, PLZ # and have a similar capacity # Create a copy of all power plants from MaStR @@ -231,18 +265,21 @@ def match_storage_units( # If a plant could be matched, add this to matched if len(selected) > 0: - matched = matched.append( - gpd.GeoDataFrame( - data={ - "source": "MaStR scaled with NEP 2021 list", - "MaStRNummer": selected.EinheitMastrNummer.head(1), - "carrier": carrier, - "el_capacity": row.c2035_capacity, - "scenario": "eGon2035", - "geometry": selected.geometry.head(1), - "voltage_level": selected.voltage_level.head(1), - } - ) + matched = pd.concat( + [ + matched, + gpd.GeoDataFrame( + data={ + "source": "MaStR scaled with NEP 2021 list", + "MaStRNummer": selected.EinheitMastrNummer.head(1), + "carrier": carrier, + "el_capacity": row.elec_capacity, + "scenario": scn, + "geometry": selected.geometry.head(1), + "voltage_level": selected.voltage_level.head(1), + } + ), + ] ) # Drop matched storage units from nep @@ -296,13 +333,13 @@ def get_location(unmatched): # Copy units with lon and lat to a new dataframe located = unmatched[ - ["bnetza_id", "name", "carrier", "city", "c2035_capacity", "geometry"] + ["bnetza_id", "name", "carrier", "city", "elec_capacity", "geometry"] ].copy() located.dropna(subset=["geometry"], inplace=True) # Rename columns for compatibility reasons located = located.rename( - columns={"c2035_capacity": "el_capacity", "bnetza_id": "MaStRNummer"} + columns={"elec_capacity": "el_capacity", "bnetza_id": "MaStRNummer"} ) located["scenario"] = "eGon2035" located["source"] = "NEP power plants geolocated using city" @@ -330,7 +367,7 @@ def apply_voltage_level_thresholds(power_plants): # account which were defined in the eGon project. Existing entries on voltage # will be overwritten - power_plants.loc[power_plants["el_capacity"] < 0.1, "voltage_level"] = 7 + power_plants.loc[power_plants["el_capacity"] <= 0.1, "voltage_level"] = 7 power_plants.loc[power_plants["el_capacity"] > 0.1, "voltage_level"] = 6 power_plants.loc[power_plants["el_capacity"] > 0.2, "voltage_level"] = 5 power_plants.loc[power_plants["el_capacity"] > 5.5, "voltage_level"] = 4 diff --git a/src/egon/data/datasets/storages_etrago/__init__.py b/src/egon/data/datasets/storages_etrago/__init__.py index 3382c62ff..5f2793629 100644 --- a/src/egon/data/datasets/storages_etrago/__init__.py +++ b/src/egon/data/datasets/storages_etrago/__init__.py @@ -6,11 +6,9 @@ import geopandas as gpd import pandas as pd from egon.data import db, config -import egon.data.datasets.scenario_parameters.parameters as scenario_parameters from egon.data.datasets import Dataset from egon.data.datasets.scenario_parameters import ( get_sector_parameters, - EgonScenario, ) @@ -18,21 +16,20 @@ class StorageEtrago(Dataset): def __init__(self, dependencies): super().__init__( name="StorageEtrago", - version="0.0.8", + version="0.0.9", dependencies=dependencies, tasks=(insert_PHES, extendable_batteries), ) def insert_PHES(): - # Get datasets configuration sources = config.datasets()["storage_etrago"]["sources"] targets = config.datasets()["storage_etrago"]["targets"] engine = db.engine() - scenario = ["eGon2035", "eGon100RE"] + scenario = config.settings()["egon-data"]["--scenarios"] for scn in scenario: # Delete outdated data on pumped hydro units (PHES) inside Germany from database db.execute_sql( @@ -59,14 +56,15 @@ def insert_PHES(): next_bus_id = db.next_etrago_id("storage") # Add missing PHES specific information suitable for eTraGo selected from scenario_parameter table - parameters = scenario_parameters.electricity(scn)["efficiency"][ - "pumped_hydro" - ] + parameters = get_sector_parameters( + "electricity", scn + )["efficiency"]["pumped_hydro"] phes["storage_id"] = range(next_bus_id, next_bus_id + len(phes)) phes["max_hours"] = parameters["max_hours"] phes["efficiency_store"] = parameters["store"] phes["efficiency_dispatch"] = parameters["dispatch"] phes["standing_loss"] = parameters["standing_loss"] + phes["cyclic_state_of_charge"] = parameters["cyclic_state_of_charge"] # Write data to db phes.to_sql( @@ -79,7 +77,6 @@ def insert_PHES(): def extendable_batteries_per_scenario(scenario): - # Get datasets configuration sources = config.datasets()["storage_etrago"]["sources"] targets = config.datasets()["storage_etrago"]["targets"] @@ -156,6 +153,10 @@ def extendable_batteries_per_scenario(scenario): "electricity", scenario )["efficiency"]["battery"]["standing_loss"] + extendable_batteries["cyclic_state_of_charge"] = get_sector_parameters( + "electricity", scenario + )["efficiency"]["battery"]["cyclic_state_of_charge"] + extendable_batteries["carrier"] = "battery" # Merge dataframes to fill p_nom_min column @@ -177,6 +178,5 @@ def extendable_batteries_per_scenario(scenario): def extendable_batteries(): - - extendable_batteries_per_scenario("eGon2035") - extendable_batteries_per_scenario("eGon100RE") + for scn in config.settings()["egon-data"]["--scenarios"]: + extendable_batteries_per_scenario(scn) diff --git a/src/egon/data/datasets/substation/__init__.py b/src/egon/data/datasets/substation/__init__.py index 594ed40f9..122d3b687 100644 --- a/src/egon/data/datasets/substation/__init__.py +++ b/src/egon/data/datasets/substation/__init__.py @@ -1,11 +1,10 @@ """The central module containing code to create substation tables """ -from airflow.operators.postgres_operator import PostgresOperator from geoalchemy2.types import Geometry from sqlalchemy import Column, Float, Integer, Sequence, Text from sqlalchemy.ext.declarative import declarative_base -import importlib_resources as resources +import os from egon.data import db from egon.data.datasets import Dataset @@ -80,22 +79,8 @@ def __init__(self, dependencies): create_tables, create_sql_functions, { - PostgresOperator( - task_id="hvmv_substation", - sql=resources.read_text( - __name__, "hvmv_substation.sql" - ), - postgres_conn_id="egon_data", - autocommit=True, - ), - PostgresOperator( - task_id="ehv_substation", - sql=resources.read_text( - __name__, "ehv_substation.sql" - ), - postgres_conn_id="egon_data", - autocommit=True, - ), + extract_hvmv, + extract_ehv, }, transfer_busses, ), @@ -237,10 +222,7 @@ def create_sql_functions(): def transfer_busses(): - - targets = egon.data.config.datasets()["substation_extraction"][ - "targets" - ] + targets = egon.data.config.datasets()["substation_extraction"]["targets"] db.execute_sql( f""" @@ -254,4 +236,13 @@ def transfer_busses(): ref, operator, dbahn, status FROM {targets['hvmv_substation']['schema']}. {targets['hvmv_substation']['table']} ORDER BY osm_id) as foo; - """) + """ + ) + + +def extract_ehv(): + db.execute_sql_script(os.path.dirname(__file__) + "/ehv_substation.sql") + + +def extract_hvmv(): + db.execute_sql_script(os.path.dirname(__file__) + "/hvmv_substation.sql") diff --git a/src/egon/data/datasets/zensus_vg250.py b/src/egon/data/datasets/zensus_vg250.py index f055f0b8d..e5d39906e 100755 --- a/src/egon/data/datasets/zensus_vg250.py +++ b/src/egon/data/datasets/zensus_vg250.py @@ -15,14 +15,18 @@ ) from sqlalchemy.ext.declarative import declarative_base import geopandas as gpd +import pandas as pd from egon.data import db from egon.data.datasets import Dataset from egon.data.datasets.vg250 import vg250_metadata_resources_fields from egon.data.metadata import ( context, + generate_resource_fields_from_sqla_model, + license_ccby, licenses_datenlizenz_deutschland, meta_metadata, + sources, ) import egon.data.config @@ -33,7 +37,7 @@ class ZensusVg250(Dataset): def __init__(self, dependencies): super().__init__( name="ZensusVg250", - version="0.0.2", + version="0.0.3", dependencies=dependencies, tasks=( map_zensus_vg250, @@ -41,6 +45,7 @@ def __init__(self, dependencies): add_metadata_zensus_inside_ger, population_in_municipalities, add_metadata_vg250_gem_pop, + add_metadata_vg250_zensus, ), ) @@ -213,7 +218,7 @@ def map_zensus_vg250(): join_missing = gpd.sjoin( missing_cells, boundaries_buffer, how="inner", op="intersects" ) - join = join.append(join_missing) + join = pd.concat([join, join_missing]) missing_cells = gdf[ (~gdf.id.isin(join.id_left)) & (gdf.population > 0) ] @@ -653,3 +658,95 @@ def add_metadata_vg250_gem_pop(): Vg250GemPopulation.__table__.schema, Vg250GemPopulation.__table__.name, ) + + +def add_metadata_vg250_zensus(): + # Import column names and datatypes + fields = [ + { + "name": "zensus_population_id", + "description": "Index of zensus population cell", + "type": "integer", + "unit": "none", + }, + { + "name": "zensus_geom", + "description": "Gemetry of zensus cell", + "type": "geometry(point,3035)", + "unit": "none", + }, + { + "name": "vg250_municipality_id", + "description": "Index of municipality", + "type": "integer", + "unit": "none", + }, + { + "name": "vg250_nuts3", + "description": "NUTS3 region-code", + "type": "varchar", + "unit": "none", + }, + ] + + meta = { + "name": "boundaries.egon_map_zensus_vg250", + "title": "eGon feedin timeseries for RES", + "id": "WILL_BE_SET_AT_PUBLICATION", + "description": "Weather-dependent feedin timeseries for RES", + "language": ["EN"], + "publicationDate": datetime.date.today().isoformat(), + "context": context(), + "spatial": { + "location": None, + "extent": "Germany", + "resolution": "100mx100m", + }, + "sources": [ + sources()["zensus"], + sources()["vg250"], + sources()["egon-data"], + ], + "licenses": [ + license_ccby( + "© Bundesamt für Kartographie und Geodäsie 2020 (Daten verändert); " + "© Statistische Ämter des Bundes und der Länder 2014 " + "© Jonathan Amme, Clara Büttner, Ilka Cußmann, Julian Endres, Carlos Epia, Stephan Günther, Ulf Müller, Amélia Nadal, Guido Pleßmann, Francesco Witte", + ) + ], + "contributors": [ + { + "title": "Clara Büttner", + "email": "http://github.com/ClaraBuettner", + "date": time.strftime("%Y-%m-%d"), + "object": None, + "comment": "Added metadata", + }, + ], + "resources": [ + { + "profile": "tabular-data-resource", + "name": "boundaries.egon_map_zensus_vg250", + "path": None, + "format": "PostgreSQL", + "encoding": "UTF-8", + "schema": { + "fields": fields, + "primaryKey": ["index"], + "foreignKeys": [], + }, + "dialect": {"delimiter": None, "decimalSeparator": "."}, + } + ], + "metaMetadata": meta_metadata(), + } + + # Create json dump + meta_json = "'" + json.dumps(meta) + "'" + + # Add metadata as a comment to the table + db.submit_comment( + meta_json, + MapZensusVg250.__table__.schema, + MapZensusVg250.__table__.name, + ) diff --git a/src/egon/data/datasets_original.yml b/src/egon/data/datasets_original.yml new file mode 100755 index 000000000..beb79027d --- /dev/null +++ b/src/egon/data/datasets_original.yml @@ -0,0 +1,1234 @@ +openstreetmap: + original_data: + source: + url: "https://download.geofabrik.de/europe/germany-220101.osm.pbf" + url_testmode: "https://download.geofabrik.de/europe/germany/schleswig-holstein-220101.osm.pbf" + stylefile: "oedb.style" + target: + table_prefix: "osm" + file: "germany-220101.osm.pbf" + file_testmode: "schleswig-holstein-220101.osm.pbf" + processed: + schema: "openstreetmap" + tables: + - "osm_line" + - "osm_nodes" + - "osm_point" + - "osm_polygon" + - "osm_rels" + - "osm_roads" + - "osm_ways" +vg250: + original_data: + source: + url: "https://daten.gdz.bkg.bund.de/produkte/vg/vg250_ebenen_0101/2020/vg250_01-01.geo84.shape.ebenen.zip" + target: + file: "vg250_01-01.geo84.shape.ebenen.zip" + processed: + schema: "boundaries" + file_table_map: + "VG250_STA.shp": "vg250_sta" + "VG250_LAN.shp": "vg250_lan" + "VG250_RBZ.shp": "vg250_rbz" + "VG250_KRS.shp": "vg250_krs" + "VG250_VWG.shp": "vg250_vwg" + "VG250_GEM.shp": "vg250_gem" + +bgr: + processed: + schema: "boundaries" + file_table_map: + "Potenzialbewertung_InSpEE_InSpEE_DS.shp": "inspee_saltstructures" + sources: + vg250_federal_states: + schema: 'boundaries' + table: 'vg250_lan' + saltcaverns: + schema: 'boundaries' + table: 'inspee_saltstructures' + targets: + storage_potential: + schema: 'grid' + table: 'egon_saltstructures_storage_potential' + +zensus_population: + original_data: + source: + url: "https://www.zensus2011.de/SharedDocs/Downloads/DE/Pressemitteilung/DemografischeGrunddaten/csv_Bevoelkerung_100m_Gitter.zip?__blob=publicationFile&v=3" + target: + file: "csv_Bevoelkerung_100m_Gitter.zip" + processed: + schema: "society" + table: "destatis_zensus_population_per_ha" +zensus_misc: + original_data: + source: + url: + - "https://www.zensus2011.de/SharedDocs/Downloads/DE/Pressemitteilung/DemografischeGrunddaten/csv_Haushalte_100m_Gitter.zip?__blob=publicationFile&v=2" + - "https://www.zensus2011.de/SharedDocs/Downloads/DE/Pressemitteilung/DemografischeGrunddaten/csv_Gebaeude_100m_Gitter.zip?__blob=publicationFile&v=2" + - "https://www.zensus2011.de/SharedDocs/Downloads/DE/Pressemitteilung/DemografischeGrunddaten/csv_Wohnungen_100m_Gitter.zip?__blob=publicationFile&v=5" + processed: + schema: "society" + file_table_map: + "csv_Haushalte_100m_Gitter.zip": "egon_destatis_zensus_household_per_ha" + "csv_Gebaeude_100m_Gitter.zip": "egon_destatis_zensus_building_per_ha" + "csv_Wohnungen_100m_Gitter.zip": "egon_destatis_zensus_apartment_per_ha" + +map_zensus_vg250: + sources: + zensus_population: + schema: 'society' + table: 'destatis_zensus_population_per_ha' + vg250_municipalities: + schema: 'boundaries' + table: 'vg250_gem' + targets: + map: + schema: 'boundaries' + table: 'egon_map_zensus_vg250' + +demandregio_installation: + sources: + git-repository: "https://github.com/openego/disaggregator.git" + branch: "features/pip_install" + targets: + path: 'demandregio-disaggregator' + +demandregio_society: + sources: + disaggregator: + path: 'demandregio-disaggregator' + vg250_krs: + schema: 'boundaries' + table: 'vg250_krs' + scenarios: + schema: 'scenario' + table: 'egon_scenario_parameters' + targets: + population: + schema: 'society' + table: 'egon_demandregio_population' + household: + schema: 'society' + table: 'egon_demandregio_household' + + +demandregio_household_demand: + sources: + disaggregator: + path: 'demandregio-disaggregator' + vg250_krs: + schema: 'boundaries' + table: 'vg250_krs' + scenarios: + schema: 'scenario' + table: 'egon_scenario_parameters' + targets: + household_demand: + schema: 'demand' + table: 'egon_demandregio_hh' + +demandregio_cts_ind_demand: + sources: + disaggregator: + path: 'demandregio-disaggregator' + vg250_krs: + schema: 'boundaries' + table: 'vg250_krs' + new_consumers_2035: 'new_largescale_consumers_nep.csv' + new_consumers_2050: + "pes-demand-today": "industrial_energy_demand_per_country_today.csv" + "pes-production-tomorrow": "industrial_production_per_country_tomorrow_2050.csv" + "pes-sector-ratios": "industry_sector_ratios.csv" + wz_definitions: + "CTS": 'CTS_WZ_definition.csv' + "industry": 'ind_WZ_definition.csv' + scenarios: + schema: 'scenario' + table: 'egon_scenario_parameters' + targets: + cts_ind_demand: + schema: 'demand' + table: 'egon_demandregio_cts_ind' + wz_definitions: + schema: 'demand' + table: 'egon_demandregio_wz' + timeseries_cts_ind: + schema: 'demand' + table: 'egon_demandregio_timeseries_cts_ind' + +electrical_demands_households: + sources: + demandregio: + schema: 'demand' + table: 'egon_demandregio_hh' + scenarios: ["eGon2035", "eGon100RE"] + population_prognosis_zensus: + schema: 'society' + table: 'egon_population_prognosis' + map_zensus_vg250: + schema: 'boundaries' + table: 'egon_map_zensus_vg250' + targets: + household_demands_zensus: + schema: 'demand' + table: 'egon_demandregio_zensus_electricity' + +electrical_demands_cts: + sources: + demandregio: + schema: 'demand' + table: 'egon_demandregio_cts_ind' + scenarios: ["eGon2035", "eGon100RE"] + demandregio_wz: + schema: 'demand' + table: 'egon_demandregio_wz' + heat_demand_cts: + schema: 'demand' + table: 'egon_peta_heat' + map_zensus_vg250: + schema: 'boundaries' + table: 'egon_map_zensus_vg250' + targets: + cts_demands_zensus: + schema: 'demand' + table: 'egon_demandregio_zensus_electricity' + +substation_extraction: + sources: + osm_ways: + schema: 'openstreetmap' + table: 'osm_ways' + osm_nodes: + schema: 'openstreetmap' + table: 'osm_nodes' + osm_points: + schema: 'openstreetmap' + table: 'osm_point' + osm_lines: + schema: 'openstreetmap' + table: 'osm_line' + targets: + hvmv_substation: + schema: 'grid' + table: 'egon_hvmv_transfer_buses' + ehv_substation: + schema: 'grid' + table: 'egon_ehv_transfer_buses' + transfer_busses: + table: 'transfer_busses_complete' + +substation_voronoi: + sources: + hvmv_substation: + schema: 'grid' + table: 'egon_hvmv_substation' + ehv_substation: + schema: 'grid' + table: 'egon_ehv_substation' + boundaries: + schema: 'boundaries' + table: 'vg250_sta_union' + targets: + hvmv_substation_voronoi: + schema: 'grid' + table: 'egon_hvmv_substation_voronoi' + ehv_substation_voronoi: + schema: 'grid' + table: 'egon_ehv_substation_voronoi' + +society_prognosis: + soucres: + zensus_population: + schema: 'society' + table: 'destatis_zensus_population_per_ha' + zensus_households: + schema: 'society' + table: 'egon_destatis_zensus_household_per_ha' + demandregio_population: + schema: 'society' + table: 'egon_demandregio_population' + demandregio_households: + schema: 'society' + table: 'egon_demandregio_household' + map_zensus_vg250: + schema: 'boundaries' + table: 'egon_map_zensus_vg250' + + target: + population_prognosis: + schema: 'society' + table: 'egon_population_prognosis' + household_prognosis: + schema: 'society' + table: 'egon_household_prognosis' + +scenario_input: + sources: + eGon2035: + capacities: "NEP2035_V2021_scnC2035.xlsx" + list_conv_pp: "Kraftwerksliste_NEP_2021_konv.csv" + eGon100RE: + capacities: "nodal_capacities.csv" + boundaries: + schema: 'boundaries' + table: 'vg250_lan' + zensus_population: + schema: 'society' + table: 'destatis_zensus_population_per_ha' + targets: + scenario_capacities: + schema: 'supply' + table: 'egon_scenario_capacities' + nep_conventional_powerplants: + schema: 'supply' + table: 'egon_nep_2021_conventional_powerplants' + +mastr: + technologies: + - "wind" + - "hydro" + - "solar" + - "biomass" + - "combustion" + - "nuclear" + - "gsgk" + - "storage" + file_basename: "bnetza_mastr" + deposit_id: 10480930 + +mastr_new: + technologies: + - "wind" + - "hydro" + - "solar" + - "biomass" + file_basename: "bnetza_mastr" + deposit_id: 10480958 + +re_potential_areas: + target: + schema: "supply" + path_table_map: + "potentialarea_pv_agriculture.gpkg": "egon_re_potential_area_pv_agriculture" + "potentialarea_pv_road_railway.gpkg": "egon_re_potential_area_pv_road_railway" + "potentialarea_wind.gpkg": "egon_re_potential_area_wind" + path_table_map_testmode: + "potentialarea_pv_agriculture_SH.gpkg": "egon_re_potential_area_pv_agriculture" + "potentialarea_pv_road_railway_SH.gpkg": "egon_re_potential_area_pv_road_railway" + "potentialarea_wind_SH.gpkg": "egon_re_potential_area_wind" + +peta5_0_1_res_heat_demands: + original_data: + source: + # In case the link breaks (hopefully it does not), you are able to download + # the data manually. Here you also find background information: + # https://s-eenergies-open-data-euf.hub.arcgis.com/search?categories=seenergies_buildings + # https://www.arcgis.com/home/item.html?id=d7d18b63250240a49eb81db972aa573e + url: "https://arcgis.com/sharing/rest/content/items/d7d18b63250240a49eb81db972aa573e/data" + target: + path: "Peta5_0_1_HD_res.zip" + processed: + schema: "demand" + file_table_map: + table: # 2015 heat demand data not stored in database +peta5_0_1_ser_heat_demands: + original_data: + source: + # In case the link breaks (hopefully it does not), you are able to download + # the data manually. Here you also find background information: + # https://s-eenergies-open-data-euf.hub.arcgis.com/search?categories=seenergies_buildings + # https://www.arcgis.com/home/item.html?id=52ff5e02111142459ed5c2fe3d80b3a0 + url: "https://arcgis.com/sharing/rest/content/items/52ff5e02111142459ed5c2fe3d80b3a0/data" + target: + path: "Peta5_0_1_HD_ser.zip" + processed: + schema: "demand" + file_table_map: + table: # 2015 heat demand data not stored in database + +hotmaps_current_policy_scenario_heat_demands_buildings: + sources: + url: "https://gitlab.com/hotmaps/scen_current_building_demand/-/raw/master/data/scen_current_building_demand.csv?inline=false" + targets: + path: "scen_current_building_demand.csv" + +power_plants: + sources: + mastr_pv: "bnetza_mastr_solar_cleaned.csv" + mastr_wind: "bnetza_mastr_wind_cleaned.csv" + mastr_biomass: "bnetza_mastr_biomass_cleaned.csv" + mastr_hydro: "bnetza_mastr_hydro_cleaned.csv" + mastr_location: "location_elec_generation_raw.csv" + mastr_combustion_without_chp: "supply.egon_mastr_conventional_without_chp" + mastr_storage: "bnetza_mastr_storage_cleaned.csv" + mastr_gsgk: "bnetza_mastr_gsgk_cleaned.csv" + capacities: "supply.egon_scenario_capacities" + geom_germany: "boundaries.vg250_sta_union" + geom_federal_states: "boundaries.vg250_lan" + egon_mv_grid_district: "grid.egon_mv_grid_district" + ehv_voronoi: "grid.egon_ehv_substation_voronoi" + nep_conv: "supply.egon_nep_2021_conventional_powerplants" + buses_data: "osmtgmod_results.bus_data" + power_plants: 'supply.egon_power_plants' + nep_2035: "NEP2035_V2021_scnC2035.xlsx" + target: + table: 'egon_power_plants' + schema: 'supply' + +storages: + sources: + mastr_storage: "bnetza_mastr_storage_cleaned.csv" + capacities: "supply.egon_scenario_capacities" + geom_germany: "boundaries.vg250_sta_union" + geom_federal_states: "boundaries.vg250_lan" + egon_mv_grid_district: "grid.egon_mv_grid_district" + ehv_voronoi: "grid.egon_ehv_substation_voronoi" + nep_conv: "supply.egon_nep_2021_conventional_powerplants" + nep_capacities: "NEP2035_V2021_scnC2035.xlsx" + generators: "grid.egon_etrago_generator" + bus: "grid.egon_etrago_bus" + target: + table: 'egon_storages' + schema: 'supply' + +landuse: + sources: + osm_polygons: + schema: 'openstreetmap' + table: 'osm_polygon' + vg250: + schema: 'boundaries' + table: 'vg250_sta_union' + target: + table: 'osm_landuse' + schema: 'openstreetmap' + +era5_weather_data: + targets: + weather_data: + path: 'cutouts' + weather_cells: + schema: 'supply' + table: 'egon_era5_weather_cells' + +renewable_feedin: + sources: + era5_weather_data: + path: 'cutouts' + weather_cells: + schema: 'supply' + table: 'egon_era5_weather_cells' + vg250_lan_union: + schema: 'boundaries' + table: 'vg250_lan_union' + vg250_sta_union: + schema: 'boundaries' + table: 'vg250_sta_union' + targets: + feedin_table: + schema: 'supply' + table: 'egon_era5_renewable_feedin' + +map_zensus_grid_districts: + sources: + zensus_population: + schema: 'society' + table: 'destatis_zensus_population_per_ha' + egon_mv_grid_district: + schema: 'grid' + table: 'egon_mv_grid_district' + targets: + map: + schema: 'boundaries' + table: 'egon_map_zensus_grid_districts' + +electrical_load_curves_cts: + sources: + map_grid_districts: + schema: 'boundaries' + table: 'egon_map_zensus_grid_districts' + map_vg250: + schema: 'boundaries' + table: 'egon_map_zensus_vg250' + zensus_electricity: + schema: 'demand' + table: 'egon_demandregio_zensus_electricity' + demandregio_cts: + schema: 'demand' + table: 'egon_demandregio_cts_ind' + demandregio_wz: + schema: 'demand' + table: 'egon_demandregio_wz' + demandregio_timeseries: + schema: 'demand' + table: 'egon_demandregio_timeseries_cts_ind' + hvmv_substation: + schema: 'grid' + table: 'egon_hvmv_substation' + targets: + cts_demand_curves: + schema: 'demand' + table: 'egon_etrago_electricity_cts' + +hh_demand_profiles: + sources: + household_electricity_demand_profiles: + path_testmode: "hh_el_load_profiles_2511.hdf" + path: "hh_el_load_profiles_100k.hdf" + + zensus_household_types: + path: "Zensus2011_Personen.csv" + +map_mvgrid_vg250: + sources: + egon_mv_grid_district: + schema: 'grid' + table: 'egon_mv_grid_district' + federal_states: + schema: 'boundaries' + table: 'vg250_lan_union' + targets: + map: + schema: 'boundaries' + table: 'egon_map_mvgriddistrict_vg250' + +solar_rooftop: + sources: + solar_feedin: + schema: 'supply' + table: 'egon_era5_renewable_feedin' + egon_mv_grid_district: + schema: 'grid' + table: 'egon_mv_grid_district' + weather_cells: + schema: 'supply' + table: 'egon_era5_weather_cells' + electricity_demand: + schema: 'demand' + table: 'egon_demandregio_zensus_electricity' + map_zensus_grid_districts: + schema: 'boundaries' + table: 'egon_map_zensus_grid_districts' + map_grid_boundaries: + schema: 'boundaries' + table: 'egon_map_mvgriddistrict_vg250' + scenario_capacities: + schema: 'supply' + table: 'egon_scenario_capacities' + federal_states: + schema: 'boundaries' + table: 'vg250_lan' + targets: + generators: + schema: 'grid' + table: 'egon_etrago_generator' + generator_timeseries: + schema: 'grid' + table: 'egon_etrago_generator_timeseries' + +data-bundle: + sources: + zenodo: + deposit_id: 1095882 + targets: + file: + 'data_bundle_egon_data.zip' + +pypsa-technology-data: + sources: + zenodo: + deposit_id: 5544025 + file: 'PyPSA/technology-data-v0.3.0.zip' + targets: + file: + 'pypsa_technology_data_egon_data.zip' + data_dir: + 'PyPSA-technology-data-94085a8/outputs/' + +heat_supply: + sources: + scenario_capacities: + schema: 'supply' + table: 'egon_scenario_capacities' + district_heating_areas: + schema: 'demand' + table: 'egon_district_heating_areas' + chp: + schema: 'supply' + table: 'egon_chp_plants' + federal_states: + schema: 'boundaries' + table: 'vg250_lan' + heat_demand: + schema: 'demand' + table: 'egon_peta_heat' + map_zensus_grid: + schema: 'boundaries' + table: 'egon_map_zensus_grid_districts' + map_vg250_grid: + schema: 'boundaries' + table: 'egon_map_mvgriddistrict_vg250' + mv_grids: + schema: 'grid' + table: 'egon_mv_grid_district' + map_dh: + schema: 'demand' + table: 'egon_map_zensus_district_heating_areas' + etrago_buses: + table: 'egon_etrago_bus' + schema: 'grid' + targets: + district_heating_supply: + schema: 'supply' + table: 'egon_district_heating' + individual_heating_supply: + schema: 'supply' + table: 'egon_individual_heating' + +etrago_heat: + sources: + scenario_capacities: + schema: 'supply' + table: 'egon_scenario_capacities' + district_heating_areas: + schema: 'demand' + table: 'egon_district_heating_areas' + map_district_heating_areas: + schema: 'demand' + table: 'egon_map_zensus_district_heating_areas' + mv_grids: + schema: 'grid' + table: 'egon_mv_grid_district' + district_heating_supply: + schema: 'supply' + table: 'egon_district_heating' + individual_heating_supply: + schema: 'supply' + table: 'egon_individual_heating' + weather_cells: + schema: 'supply' + table: 'egon_era5_weather_cells' + feedin_timeseries: + schema: 'supply' + table: 'egon_era5_renewable_feedin' + egon_mv_grid_district: + schema: 'grid' + table: 'egon_mv_grid_district' + heat_demand: + schema: 'demand' + table: 'egon_peta_heat' + ch4_voronoi: + schema: 'grid' + table: 'egon_gas_voronoi' + targets: + heat_buses: + schema: 'grid' + table: 'egon_etrago_bus' + heat_generators: + schema: 'grid' + table: 'egon_etrago_generator' + heat_generator_timeseries: + schema: 'grid' + table: 'egon_etrago_generator_timeseries' + heat_links: + schema: 'grid' + table: 'egon_etrago_link' + heat_link_timeseries: + schema: 'grid' + table: 'egon_etrago_link_timeseries' + heat_stores: + schema: 'grid' + table: 'egon_etrago_store' + +etrago_hydrogen: + sources: + saltcavern_data: + schema: 'grid' + table: 'egon_saltstructures_storage_potential' + buses: + schema: 'grid' + table: 'egon_etrago_bus' + H2_AC_map: + schema: 'grid' + table: 'egon_etrago_ac_h2' + targets: + hydrogen_buses: + schema: 'grid' + table: 'egon_etrago_bus' + hydrogen_links: + schema: 'grid' + table: 'egon_etrago_link' + hydrogen_stores: + schema: 'grid' + table: 'egon_etrago_store' + +industrial_sites: + sources: + hotmaps: + url: "https://gitlab.com/hotmaps/industrial_sites/industrial_sites_Industrial_Database/-/raw/388278c6df35889b1447a959fc3759e3d78bf659/data/Industrial_Database.csv?inline=false" + path: "data_Industrial_Database.csv" + seenergies: + url: "https://opendata.arcgis.com/datasets/5e36c0af918040ed936b4e4c101f611d_0.csv" + path: "D5_1_Industry_Dataset_With_Demand_Data.csv" + schmidt: + path: "MA_Schmidt_Industriestandorte_georef.csv" + targets: + hotmaps: + schema: "demand" + table: "egon_hotmaps_industrial_sites" + seenergies: + schema: "demand" + table: "egon_seenergies_industrial_sites" + schmidt: + schema: "demand" + table: "egon_schmidt_industrial_sites" + sites: + schema: "demand" + table: "egon_industrial_sites" + +distributed_industrial_demand: + sources: + demandregio: + schema: 'demand' + table: 'egon_demandregio_cts_ind' + scenarios: ["eGon2021", "eGon2035", "eGon100RE"] + wz: + schema: 'demand' + table: 'egon_demandregio_wz' + osm_landuse: + schema: 'openstreetmap' + table: 'osm_landuse' + industrial_sites: + schema: 'demand' + table: 'egon_industrial_sites' + vg250_krs: + schema: 'boundaries' + table: 'vg250_krs' + targets: + osm: + schema: 'demand' + table: 'egon_demandregio_osm_ind_electricity' + sites: + schema: 'demand' + table: 'egon_demandregio_sites_ind_electricity' + +electrical_load_curves_industry: + sources: + osm: + schema: 'demand' + table: 'egon_demandregio_osm_ind_electricity' + osm_landuse: + schema: 'openstreetmap' + table: 'osm_landuse' + sites: + schema: 'demand' + table: 'egon_demandregio_sites_ind_electricity' + sites_geom: + schema: 'demand' + table: 'egon_industrial_sites' + demandregio_industry: + schema: 'demand' + table: 'egon_demandregio_cts_ind' + demandregio_wz: + schema: 'demand' + table: 'egon_demandregio_wz' + demandregio_timeseries: + schema: 'demand' + table: 'egon_demandregio_timeseries_cts_ind' + hvmv_substation: + schema: 'grid' + table: 'egon_hvmv_substation' + egon_mv_grid_district: + schema: 'grid' + table: 'egon_mv_grid_district' + egon_ehv_voronoi: + schema: 'grid' + table: 'egon_ehv_substation_voronoi' + targets: + osm_load: + schema: 'demand' + table: 'egon_osm_ind_load_curves' + osm_load_individual: + schema: 'demand' + table: 'egon_osm_ind_load_curves_individual' + sites_load: + schema: 'demand' + table: 'egon_sites_ind_load_curves' + sites_load_individual: + schema: 'demand' + table: 'egon_sites_ind_load_curves_individual' + +etrago_electricity: + sources: + etrago_buses: + table: 'egon_etrago_bus' + schema: 'grid' + osm_curves: + schema: 'demand' + table: 'egon_osm_ind_load_curves' + sites_curves: + schema: 'demand' + table: 'egon_sites_ind_load_curves' + cts_curves: + schema: 'demand' + table: 'egon_etrago_electricity_cts' + household_curves: + schema: 'demand' + table: 'egon_etrago_electricity_households' + targets: + etrago_load: + schema: 'grid' + table: 'egon_etrago_load' + etrago_load_curves: + schema: 'grid' + table: 'egon_etrago_load_timeseries' + +chp_location: + sources: + list_conv_pp: + table: 'egon_nep_2021_conventional_powerplants' + schema: 'supply' + mastr_combustion: 'bnetza_mastr_combustion_cleaned.csv' + mastr_location: 'location_elec_generation_raw.csv' + egon_mv_grid_district: 'grid.egon_mv_grid_district' + ehv_voronoi: "grid.egon_ehv_substation_voronoi" + etrago_buses: + table: 'egon_etrago_bus' + schema: 'grid' + osm_landuse: + table: 'osm_landuse' + schema: 'openstreetmap' + osm_polygon: + table: 'osm_polygon' + schema: 'openstreetmap' + district_heating_areas: + schema: 'demand' + table: 'egon_district_heating_areas' + industrial_demand_osm: + schema: 'demand' + table: 'egon_demandregio_osm_ind_electricity' + vg250_lan: + schema: 'boundaries' + table: 'vg250_lan' + mastr_biomass: "bnetza_mastr_biomass_cleaned.csv" + targets: + chp_table: + table: 'egon_chp_plants' + schema: 'supply' + mastr_conventional_without_chp: + table: 'egon_mastr_conventional_without_chp' + schema: 'supply' + +chp_etrago: + sources: + chp_table: + table: 'egon_chp_plants' + schema: 'supply' + district_heating_areas: + schema: 'demand' + table: 'egon_district_heating_areas' + etrago_buses: + table: 'egon_etrago_bus' + schema: 'grid' + targets: + link: + schema: 'grid' + table: 'egon_etrago_link' + generator: + schema: 'grid' + table: 'egon_etrago_generator' + +DSM_CTS_industry: + sources: + cts_loadcurves: + schema: 'demand' + table: 'egon_etrago_electricity_cts' + ind_osm_loadcurves: + schema: 'demand' + table: 'egon_osm_ind_load_curves' + ind_osm_loadcurves_individual: + schema: 'demand' + table: 'egon_osm_ind_load_curves_individual' + ind_sites_loadcurves: + schema: 'demand' + table: 'egon_sites_ind_load_curves' + ind_sites_loadcurves_individual: + schema: 'demand' + table: 'egon_sites_ind_load_curves_individual' + ind_sites: + schema: 'demand' + table: 'egon_industrial_sites' + ind_sites_schmidt: + schema: 'demand' + table: 'egon_schmidt_industrial_sites' + demandregio_ind_sites: + schema: 'demand' + table: 'egon_demandregio_sites_ind_electricity' + targets: + bus: + schema: 'grid' + table: 'egon_etrago_bus' + link: + schema: 'grid' + table: 'egon_etrago_link' + link_timeseries: + schema: 'grid' + table: 'egon_etrago_link_timeseries' + store: + schema: 'grid' + table: 'egon_etrago_store' + store_timeseries: + schema: 'grid' + table: 'egon_etrago_store_timeseries' + cts_loadcurves_dsm: + schema: 'demand' + table: 'egon_etrago_electricity_cts_dsm_timeseries' + ind_osm_loadcurves_individual_dsm: + schema: 'demand' + table: 'egon_osm_ind_load_curves_individual_dsm_timeseries' + demandregio_ind_sites_dsm: + schema: 'demand' + table: 'egon_demandregio_sites_ind_electricity_dsm_timeseries' + ind_sites_loadcurves_individual: + schema: 'demand' + table: 'egon_sites_ind_load_curves_individual_dsm_timeseries' + +generators_etrago: + sources: + bus: + schema: 'grid' + table: 'egon_etrago_bus' + power_plants: + schema: 'supply' + table: 'egon_power_plants' + renewable_feedin: + schema: 'supply' + table: 'egon_era5_renewable_feedin' + weather_cells: + schema: 'supply' + table: 'egon_era5_weather_cells' + egon_mv_grid_district: 'grid.egon_mv_grid_district' + ehv_voronoi: 'grid.egon_ehv_substation_voronoi' + targets: + etrago_generators: + schema: 'grid' + table: 'egon_etrago_generator' + etrago_gen_time: + schema: 'grid' + table: 'egon_etrago_generator_timeseries' + + +gas_grid: + targets: + links: + schema: 'grid' + table: 'egon_etrago_link' + buses: + schema: 'grid' + table: 'egon_etrago_bus' + + +gas_stores: + source: + buses: + schema: 'grid' + table: 'egon_etrago_bus' + target: + stores: + schema: 'grid' + table: 'egon_etrago_store' + +gas_prod: + source: + buses: + schema: 'grid' + table: 'egon_etrago_bus' + target: + stores: + schema: 'grid' + table: 'egon_etrago_generator' + +weather_BusID: + sources: + power_plants: + schema: 'supply' + table: 'egon_power_plants' + renewable_feedin: + schema: 'supply' + table: 'egon_era5_renewable_feedin' + weather_cells: + schema: 'supply' + table: 'egon_era5_weather_cells' + boundaries: + schema: 'boundaries' + table: 'vg250_sta' + egon_mv_grid_district: 'grid.egon_mv_grid_district' + ehv_voronoi: 'grid.egon_ehv_substation_voronoi' + targets: + power_plants: + schema: 'supply' + table: 'egon_power_plants' + +storage_etrago: + sources: + storage: + table: 'egon_storages' + schema: 'supply' + scenario_parameters: + table: 'egon_scenario_parameters' + schema: 'scenario' + bus: + table: 'egon_etrago_bus' + schema: 'grid' + ehv-substation: + table: 'egon_ehv_substation' + schema: 'grid' + hv-substation: + table: 'egon_hvmv_substation' + schema: 'grid' + targets: + storage: + schema: 'grid' + table: 'egon_etrago_storage' + + +dlr: + sources: + trans_lines: + schema: "grid" + table: "egon_etrago_line" + line_timeseries: + schema: "grid" + table: "egon_etrago_line_timeseries" + targets: + line_timeseries: + schema: "grid" + table: "egon_etrago_line_timeseries" + +electrical_neighbours: + sources: + electricity_buses: + schema: 'grid' + table: 'egon_etrago_bus' + lines: + schema: 'grid' + table: 'egon_etrago_line' + german_borders: + schema: 'boundaries' + table: 'vg250_sta_union' + osmtgmod_bus: + schema: 'osmtgmod_results' + table: 'bus_data' + osmtgmod_branch: + schema: 'osmtgmod_results' + table: 'branch_data' + tyndp_capacities: "TYNDP-2020-Scenario-Datafile.xlsx.zip" + tyndp_demand_2030: "Demand_TimeSeries_2030_DistributedEnergy.xlsx" + tyndp_demand_2040: "Demand_TimeSeries_2040_DistributedEnergy.xlsx" + targets: + buses: + schema: 'grid' + table: 'egon_etrago_bus' + lines: + schema: 'grid' + table: 'egon_etrago_line' + links: + schema: 'grid' + table: 'egon_etrago_link' + transformers: + schema: 'grid' + table: 'egon_etrago_transformer' + loads: + schema: 'grid' + table: 'egon_etrago_load' + load_timeseries: + schema: 'grid' + table: 'egon_etrago_load_timeseries' + generators: + schema: 'grid' + table: 'egon_etrago_generator' + generators_timeseries: + schema: 'grid' + table: 'egon_etrago_generator_timeseries' + storage: + schema: 'grid' + table: 'egon_etrago_storage' + +tyndp: + sources: + capacities: "https://2020.entsos-tyndp-scenarios.eu/wp-content/uploads/2020/06/TYNDP-2020-Scenario-Datafile.xlsx.zip" + demand_2030: "https://eepublicdownloads.entsoe.eu/tyndp-documents/2020-data/Demand_TimeSeries_2030_DistributedEnergy.xlsx" + demand_2040: "https://eepublicdownloads.entsoe.eu/tyndp-documents/2020-data/Demand_TimeSeries_2040_DistributedEnergy.xlsx" + targets: + capacities: "TYNDP-2020-Scenario-Datafile.xlsx.zip" + demand_2030: "Demand_TimeSeries_2030_DistributedEnergy.xlsx" + demand_2040: "Demand_TimeSeries_2040_DistributedEnergy.xlsx" + +gas_neighbours: + sources: + tyndp_capacities: "TYNDP-2020-Scenario-Datafile.xlsx.zip" + buses: + schema: 'grid' + table: 'egon_etrago_bus' + links: + schema: 'grid' + table: 'egon_etrago_link' + targets: + generators: + schema: 'grid' + table: 'egon_etrago_generator' + loads: + schema: 'grid' + table: 'egon_etrago_load' + load_timeseries: + schema: 'grid' + table: 'egon_etrago_load_timeseries' + stores: + schema: 'grid' + table: 'egon_etrago_store' + links: + schema: 'grid' + table: 'egon_etrago_link' + +emobility_mit: + original_data: + sources: + RS7: + url: "https://www.bmvi.de/SharedDocs/DE/Anlage/G/regiostar-referenzdateien.xlsx?__blob=publicationFile" + file: "regiostar-referenzdateien.xlsx" + file_processed: "regiostar-referenzdateien_preprocessed.csv" + sheet: "ReferenzGebietsstand2020" + KBA: + url: "https://www.kba.de/SharedDocs/Downloads/DE/Statistik/Fahrzeuge/FZ1/fz1_2021.xlsx?__blob=publicationFile&v=2" + file: "fz1_2021.xlsx" + file_processed: "fz1_2021_preprocessed.csv" + sheet: "FZ1.1" + columns: "D, J:N" + skiprows: 8 + trips: + eGon2035: + file: "eGon2035_RS7_min2k_2022-06-01_175429_simbev_run.tar.gz" + file_metadata: "metadata_simbev_run.json" + eGon100RE: + file: "eGon100RE_RS7_min2k_2022-06-01_175444_simbev_run.tar.gz" + file_metadata: "metadata_simbev_run.json" + scenario: + # used scenario variation (available scenarios see parameters.py) + variation: + eGon2035: "NEP C 2035" + eGon100RE: "Reference 2050" + # name of low-flex scenario + lowflex: + create_lowflex_scenario: True + names: + eGon2035: "eGon2035_lowflex" + eGon100RE: "eGon100RE_lowflex" + + model_timeseries: + reduce_memory: True + export_results_to_csv: True + parallel_tasks: 10 + +charging_infrastructure: + original_data: + sources: + tracbev: + url: "https://zenodo.org/record/6466480/files/data.zip?download=1" + file: "data.zip" + files_to_use: + - "hpc_positions.gpkg" + - "landuse.gpkg" + - "poi_cluster.gpkg" + - "public_positions.gpkg" + srid: 3035 + constants: + work_weight_retail: 0.8 + work_weight_commercial: 1.25 + work_weight_industrial: 1 + single_family_home_share: 0.6 + single_family_home_spots: 1.5 + multi_family_home_share: 0.4 + multi_family_home_spots: 10 + random_seed: 5 + targets: + charging_infrastructure: + schema: "grid" + table: "egon_emob_charging_infrastructure" + cols_to_export: + - "mv_grid_id" + - "use_case" + - "weight" + - "geometry" + +mobility_hgv: + original_data: + sources: + BAST: + url: "https://www.bast.de/DE/Verkehrstechnik/Fachthemen/v2-verkehrszaehlung/Daten/2020_1/Jawe2020.csv?view=renderTcDataExportCSV&cms_strTyp=A" + file: "Jawe2020.csv" + relevant_columns: ["DTV_SV_MobisSo_Q", "Koor_WGS84_E", "Koor_WGS84_N"] + srid: 4326 + germany: + url: "https://raw.githubusercontent.com/isellsoap/deutschlandGeoJSON/main/1_deutschland/1_sehr_hoch.geo.json" + NUTS: + url: "https://daten.gdz.bkg.bund.de/produkte/vg/nuts250_1231/aktuell/nuts250_12-31.utm32s.shape.zip" + file: "nuts250_12-31.utm32s.shape.zip" + shp_file: "nuts250_12-31.utm32s.shape/nuts250_1231/250_NUTS1.shp" + NUTS_CODE: "DEF" + tables: + srid: 3035 + srid_buses: 4326 + constants: + # leakage rate / losses - constant + # https://www.energy.gov/eere/fuelcells/doe-technical-targets-hydrogen-delivery + # --> 0.5% + leakage: True + leakage_rate: 0.005 + # Hydrogen + # HGV traffic hydrogen consumption in kg per 100 km + # Assumption: 30L Diesel / 100 km --> kgH2/100 km + hydrogen_consumption: 6.68 + # share of FCEV trucks + # within the egon project the flat guess is 100% + fcev_share: 1. + scenarios: + - "eGon2035" + - "eGon100RE" + carrier: "H2_hgv_load" + energy_value_h2: 39.4 # kWh/kg + hours_per_year: 8760 + fac: 0.001 + hgv_mileage: + # NEP data + # https://www.netzentwicklungsplan.de/sites/default/files/paragraphs-files/NEP_2035_V2021_2_Entwurf_Teil1.pdf + # total amount of HGVs - Scenario C 2035 + # hgv_amount = 100000 + # HGV traffic annual mileage per vehicle + # hgv_mean_mileage = 100000 + # Total mileage + eGon2035: 10000000000 + # Langfristszenarien + # https://www.langfristszenarien.de/enertile-explorer-wAssets/docs/LFS3_Langbericht_Verkehr_final.pdf#page=17 + eGon100RE: 40000000000 + +home_batteries: + constants: + scenarios: + - "eGon2035" + - "eGon100RE" + # Mean ratio between the storage capacity and the power of the pv rooftop system + cbat_ppv_ratio: 1 + rtol: 0.05 + max_it: 100 + sources: + etrago_storage: + schema: 'grid' + table: 'egon_etrago_storage' + storage: + schema: 'supply' + table: 'egon_storages' + targets: + home_batteries: + schema: 'supply' + table: 'egon_home_batteries' diff --git a/src/egon/data/metadata.py b/src/egon/data/metadata.py deleted file mode 100644 index 8ade5c733..000000000 --- a/src/egon/data/metadata.py +++ /dev/null @@ -1,272 +0,0 @@ -from egon.data.db import engine -from geoalchemy2 import Geometry -from sqlalchemy import MetaData, Table -from sqlalchemy.dialects.postgresql.base import ischema_names - - -def context(): - """ - Project context information for metadata - - Returns - ------- - dict - OEP metadata conform data license information - """ - - return { - "homepage": "https://ego-n.org/", - "documentation": "https://egon-data.readthedocs.io/en/latest/", - "sourceCode": "https://github.com/openego/eGon-data", - "contact": "https://ego-n.org/partners/", - "grantNo": "03EI1002", - "fundingAgency": "Bundesministerium für Wirtschaft und Energie", - "fundingAgencyLogo": "https://www.innovation-beratung-" - "foerderung.de/INNO/Redaktion/DE/Bilder/" - "Titelbilder/titel_foerderlogo_bmwi.jpg?" - "__blob=normal&v=3", - "publisherLogo": "https://ego-n.org/images/eGon_logo_" - "noborder_transbg.svg", - } - - -def meta_metadata(): - """ - Meta data on metadata - - Returns - ------- - dict - OEP metadata conform metadata on metadata - """ - - return { - "metadataVersion": "OEP-1.4.1", - "metadataLicense": { - "name": "CC0-1.0", - "title": "Creative Commons Zero v1.0 Universal", - "path": ("https://creativecommons.org/publicdomain/zero/1.0/"), - }, - } - - -def licenses_datenlizenz_deutschland(attribution): - """ - License information for Datenlizenz Deutschland - - Parameters - ---------- - attribution : str - Attribution for the dataset incl. © symbol, e.g. '© GeoBasis-DE / BKG' - - Returns - ------- - dict - OEP metadata conform data license information - """ - - return { - "name": "dl-by-de/2.0", - "title": "Datenlizenz Deutschland – Namensnennung – Version 2.0", - "path": "www.govdata.de/dl-de/by-2-0", - "instruction": ( - "Jede Nutzung ist unter den Bedingungen dieser „Datenlizenz " - "Deutschland - Namensnennung - Version 2.0 zulässig.\nDie " - "bereitgestellten Daten und Metadaten dürfen für die " - "kommerzielle und nicht kommerzielle Nutzung insbesondere:" - "(1) vervielfältigt, ausgedruckt, präsentiert, verändert, " - "bearbeitet sowie an Dritte übermittelt werden;\n " - "(2) mit eigenen Daten und Daten Anderer zusammengeführt und " - "zu selbständigen neuen Datensätzen verbunden werden;\n " - "(3) in interne und externe Geschäftsprozesse, Produkte und " - "Anwendungen in öffentlichen und nicht öffentlichen " - "elektronischen Netzwerken eingebunden werden.\n" - "Bei der Nutzung ist sicherzustellen, dass folgende Angaben " - "als Quellenvermerk enthalten sind:\n" - "(1) Bezeichnung des Bereitstellers nach dessen Maßgabe,\n" - "(2) der Vermerk Datenlizenz Deutschland – Namensnennung – " - "Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext " - "unter www.govdata.de/dl-de/by-2-0 sowie\n" - "(3) einen Verweis auf den Datensatz (URI)." - "Dies gilt nur soweit die datenhaltende Stelle die Angaben" - "(1) bis (3) zum Quellenvermerk bereitstellt.\n" - "Veränderungen, Bearbeitungen, neue Gestaltungen oder " - "sonstige Abwandlungen sind im Quellenvermerk mit dem Hinweis " - "zu versehen, dass die Daten geändert wurden." - ), - "attribution": attribution, - } - - -def license_odbl(attribution): - """ - License information for Open Data Commons Open Database License (ODbL-1.0) - - Parameters - ---------- - attribution : str - Attribution for the dataset incl. © symbol, e.g. - '© OpenStreetMap contributors' - - Returns - ------- - dict - OEP metadata conform data license information - """ - return { - "name": "ODbL-1.0", - "title": "Open Data Commons Open Database License 1.0", - "path": "https://opendatacommons.org/licenses/odbl/1.0/index.html", - "instruction": "You are free: To Share, To Create, To Adapt; " - "As long as you: Attribute, Share-Alike, Keep open!", - "attribution": attribution, - } - - -def license_ccby(attribution): - """ - License information for Creative Commons Attribution 4.0 International - (CC-BY-4.0) - - Parameters - ---------- - attribution : str - Attribution for the dataset incl. © symbol, e.g. '© GeoBasis-DE / BKG' - - Returns - ------- - dict - OEP metadata conform data license information - """ - return { - "name": "CC-BY-4.0", - "title": "Creative Commons Attribution 4.0 International", - "path": "https://creativecommons.org/licenses/by/4.0/legalcode", - "instruction": "You are free: To Share, To Create, To Adapt; " - "As long as you: Attribute.", - "attribution": attribution, - } - - -def license_geonutzv(attribution): - """ - License information for GeoNutzV - - Parameters - ---------- - attribution : str - Attribution for the dataset incl. © symbol, e.g. '© GeoBasis-DE / BKG' - - Returns - ------- - dict - OEP metadata conform data license information - """ - return { - "name": "geonutzv-de-2013-03-19", - "title": "Verordnung zur Festlegung der Nutzungsbestimmungen für die " - "Bereitstellung von Geodaten des Bundes", - "path": "https://www.gesetze-im-internet.de/geonutzv/", - "instruction": "Geodaten und Geodatendienste, einschließlich " - "zugehöriger Metadaten, werden für alle derzeit " - "bekannten sowie für alle zukünftig bekannten Zwecke " - "kommerzieller und nicht kommerzieller Nutzung " - "geldleistungsfrei zur Verfügung gestellt, soweit " - "durch besondere Rechtsvorschrift nichts anderes " - "bestimmt ist oder vertragliche oder gesetzliche " - "Rechte Dritter dem nicht entgegenstehen.", - "attribution": attribution, - } - - -def generate_resource_fields_from_sqla_model(model): - """ Generate a template for the resource fields for metadata from a SQL - Alchemy model. - - For details on the fields see field 14.6.1 of `Open Energy Metadata - `_ standard. - The fields `name` and `type` are automatically filled, the `description` - and `unit` must be filled manually. - - Examples - -------- - >>> from egon.data.metadata import generate_resource_fields_from_sqla_model - >>> from egon.data.datasets.zensus_vg250 import Vg250Sta - >>> resources = generate_resource_fields_from_sqla_model(Vg250Sta) - - Parameters - ---------- - model : sqlalchemy.ext.declarative.declarative_base() - SQLA model - - Returns - ------- - list of dict - Resource fields - """ - - return [ - { - "name": col.name, - "description": "", - "type": str(col.type).lower(), - "unit": "none", - } - for col in model.__table__.columns - ] - - -def generate_resource_fields_from_db_table(schema, table, geom_columns=None): - """ Generate a template for the resource fields for metadata from a - database table. - - For details on the fields see field 14.6.1 of `Open Energy Metadata - `_ standard. - The fields `name` and `type` are automatically filled, the `description` - and `unit` must be filled manually. - - Examples - -------- - >>> from egon.data.metadata import generate_resource_fields_from_db_table - >>> resources = generate_resource_fields_from_db_table( - ... 'openstreetmap', 'osm_point', ['geom', 'geom_centroid'] - ... ) # doctest: +SKIP - - Parameters - ---------- - schema : str - The target table's database schema - table : str - Database table on which to put the given comment - geom_columns : list of str - Names of all geometry columns in the table. This is required to return - Geometry data type for those columns as SQL Alchemy does not recognize - them correctly. Defaults to ['geom']. - - Returns - ------- - list of dict - Resource fields - """ - - # handle geometry columns - if geom_columns is None: - geom_columns = ["geom"] - for col in geom_columns: - ischema_names[col] = Geometry - - table = Table( - table, MetaData(), schema=schema, autoload=True, autoload_with=engine() - ) - - return [ - { - "name": col.name, - "description": "", - "type": str(col.type).lower(), - "unit": "none", - } - for col in table.c - ] diff --git a/src/egon/data/metadata/__init__.py b/src/egon/data/metadata/__init__.py new file mode 100755 index 000000000..25e267217 --- /dev/null +++ b/src/egon/data/metadata/__init__.py @@ -0,0 +1,966 @@ +from geoalchemy2 import Geometry +from omi.dialects import get_dialect +from sqlalchemy import MetaData, Table +from sqlalchemy.dialects.postgresql.base import ischema_names +import importlib_resources + +from egon.data import db, logger +from egon.data.datasets import Dataset +from egon.data.db import engine + +EGON_ATTRIBUTION: str = "© eGon development team" + + +def context(): + """ + Project context information for metadata + + Returns + ------- + dict + OEP metadata conform data license information + """ + + return { + "homepage": "https://ego-n.org/", + "documentation": "https://egon-data.readthedocs.io/en/latest/", + "sourceCode": "https://github.com/openego/eGon-data", + "contact": "https://ego-n.org/partners/", + "grantNo": "03EI1002", + "fundingAgency": "Bundesministerium für Wirtschaft und Energie", + "fundingAgencyLogo": "https://www.innovation-beratung-" + "foerderung.de/INNO/Redaktion/DE/Bilder/" + "Titelbilder/titel_foerderlogo_bmwi.jpg?" + "__blob=normal&v=3", + "publisherLogo": "https://ego-n.org/images/eGon_logo_" + "noborder_transbg.svg", + } + + +def meta_metadata(): + """ + Meta data on metadata + + Returns + ------- + dict + OEP metadata conform metadata on metadata + """ + + return { + "metadataVersion": "OEP-1.4.1", + "metadataLicense": { + "name": "CC0-1.0", + "title": "Creative Commons Zero v1.0 Universal", + "path": ("https://creativecommons.org/publicdomain/zero/1.0/"), + }, + } + + +def licenses_datenlizenz_deutschland(attribution=EGON_ATTRIBUTION): + """ + License information for Datenlizenz Deutschland + + Parameters + ---------- + attribution : str + Attribution for the dataset incl. © symbol, e.g. '© GeoBasis-DE / BKG' + + Returns + ------- + dict + OEP metadata conform data license information + """ + + return { + "name": "dl-by-de/2.0", + "title": "Datenlizenz Deutschland – Namensnennung – Version 2.0", + "path": "www.govdata.de/dl-de/by-2-0", + "instruction": ( + "Jede Nutzung ist unter den Bedingungen dieser „Datenlizenz " + "Deutschland - Namensnennung - Version 2.0 zulässig.\nDie " + "bereitgestellten Daten und Metadaten dürfen für die " + "kommerzielle und nicht kommerzielle Nutzung insbesondere:" + "(1) vervielfältigt, ausgedruckt, präsentiert, verändert, " + "bearbeitet sowie an Dritte übermittelt werden;\n " + "(2) mit eigenen Daten und Daten Anderer zusammengeführt und " + "zu selbständigen neuen Datensätzen verbunden werden;\n " + "(3) in interne und externe Geschäftsprozesse, Produkte und " + "Anwendungen in öffentlichen und nicht öffentlichen " + "elektronischen Netzwerken eingebunden werden.\n" + "Bei der Nutzung ist sicherzustellen, dass folgende Angaben " + "als Quellenvermerk enthalten sind:\n" + "(1) Bezeichnung des Bereitstellers nach dessen Maßgabe,\n" + "(2) der Vermerk Datenlizenz Deutschland – Namensnennung – " + "Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext " + "unter www.govdata.de/dl-de/by-2-0 sowie\n" + "(3) einen Verweis auf den Datensatz (URI)." + "Dies gilt nur soweit die datenhaltende Stelle die Angaben" + "(1) bis (3) zum Quellenvermerk bereitstellt.\n" + "Veränderungen, Bearbeitungen, neue Gestaltungen oder " + "sonstige Abwandlungen sind im Quellenvermerk mit dem Hinweis " + "zu versehen, dass die Daten geändert wurden." + ), + "attribution": attribution, + } + + +def license_odbl(attribution=EGON_ATTRIBUTION): + """ + License information for Open Data Commons Open Database License (ODbL-1.0) + + Parameters + ---------- + attribution : str + Attribution for the dataset incl. © symbol, e.g. + '© OpenStreetMap contributors' + + Returns + ------- + dict + OEP metadata conform data license information + """ + return { + "name": "ODbL-1.0", + "title": "Open Data Commons Open Database License 1.0", + "path": "https://opendatacommons.org/licenses/odbl/1.0/index.html", + "instruction": "You are free: To Share, To Create, To Adapt; " + "As long as you: Attribute, Share-Alike, Keep open!", + "attribution": attribution, + } + + +def license_ccby(attribution=EGON_ATTRIBUTION): + """ + License information for Creative Commons Attribution 4.0 International + (CC-BY-4.0) + + Parameters + ---------- + attribution : str + Attribution for the dataset incl. © symbol, e.g. '© GeoBasis-DE / BKG' + + Returns + ------- + dict + OEP metadata conform data license information + """ + return { + "name": "CC-BY-4.0", + "title": "Creative Commons Attribution 4.0 International", + "path": "https://creativecommons.org/licenses/by/4.0/legalcode", + "instruction": "You are free: To Share, To Create, To Adapt; " + "As long as you: Attribute.", + "attribution": attribution, + } + + +def license_geonutzv(attribution=EGON_ATTRIBUTION): + """ + License information for GeoNutzV + + Parameters + ---------- + attribution : str + Attribution for the dataset incl. © symbol, e.g. '© GeoBasis-DE / BKG' + + Returns + ------- + dict + OEP metadata conform data license information + """ + return { + "name": "geonutzv-de-2013-03-19", + "title": "Verordnung zur Festlegung der Nutzungsbestimmungen für die " + "Bereitstellung von Geodaten des Bundes", + "path": "https://www.gesetze-im-internet.de/geonutzv/", + "instruction": "Geodaten und Geodatendienste, einschließlich " + "zugehöriger Metadaten, werden für alle derzeit " + "bekannten sowie für alle zukünftig bekannten Zwecke " + "kommerzieller und nicht kommerzieller Nutzung " + "geldleistungsfrei zur Verfügung gestellt, soweit " + "durch besondere Rechtsvorschrift nichts anderes " + "bestimmt ist oder vertragliche oder gesetzliche " + "Rechte Dritter dem nicht entgegenstehen.", + "attribution": attribution, + } + + +def license_agpl(attribution=EGON_ATTRIBUTION): + """ + License information for GNU Affero General Public License v3.0 + + Parameters + ---------- + attribution : str + Attribution for the dataset incl. © symbol, e.g. '© GeoBasis-DE / BKG' + + Returns + ------- + dict + OEP metadata conform data license information + """ + return { + "name": "AGPL-3.0 License", + "title": "GNU Affero General Public License v3.0", + "path": "https://www.gnu.org/licenses/agpl-3.0.de.html", + "instruction": "Permissions of this strongest copyleft license are" + "conditioned on making available complete source code of licensed " + "works and modifications, which include larger works using a licensed" + "work, under the same license. Copyright and license notices must be" + "preserved. Contributors provide an express grant of patent rights." + "When a modified version is used to provide a service over a network," + "the complete source code of the modified version must be made " + "available.", + "attribution": attribution, + } + + +def license_dedl(attribution=EGON_ATTRIBUTION): + """ + License information for Data licence Germany – attribution – version 2.0 + + Parameters + ---------- + attribution : str + Attribution for the dataset incl. © symbol, e.g. '© GeoBasis-DE / BKG' + + Returns + ------- + dict + OEP metadata conform data license information + """ + return { + "name": "DL-DE-BY-2.0", + "title": "Data licence Germany – attribution – version 2.0", + "path": "https://www.govdata.de/dl-de/by-2-0", + "instruction": ( + "Any use will be permitted provided it fulfils the requirements of" + ' this "Data licence Germany – attribution – Version 2.0". The ' + "data and meta-data provided may, for commercial and " + "non-commercial use, in particular be copied, printed, presented, " + "altered, processed and transmitted to third parties; be merged " + "with own data and with the data of others and be combined to form" + " new and independent datasets; be integrated in internal and " + "external business processes, products and applications in public " + "and non-public electronic networks. The user must ensure that the" + " source note contains the following information: the name of the " + 'provider, the annotation "Data licence Germany – attribution – ' + 'Version 2.0" or "dl-de/by-2-0" referring to the licence text ' + "available at www.govdata.de/dl-de/by-2-0, and a reference to the " + "dataset (URI). This applies only if the entity keeping the data " + "provides the pieces of information 1-3 for the source note. " + "Changes, editing, new designs or other amendments must be marked " + "as such in the source note." + ), + "attribution": attribution, + } + + +def license_egon_data_odbl(): + """ + ODbL license with eGon data attribution + + Returns + ------- + dict + OEP metadata conform data license information for eGon tables + """ + return license_odbl("© eGon development team") + + +def generate_resource_fields_from_sqla_model(model): + """Generate a template for the resource fields for metadata from a SQL + Alchemy model. + + For details on the fields see field 14.6.1 of `Open Energy Metadata + `_ standard. + The fields `name` and `type` are automatically filled, the `description` + and `unit` must be filled manually. + + Examples + -------- + >>> from egon.data.metadata import generate_resource_fields_from_sqla_model + >>> from egon.data.datasets.zensus_vg250 import Vg250Sta + >>> resources = generate_resource_fields_from_sqla_model(Vg250Sta) + + Parameters + ---------- + model : sqlalchemy.ext.declarative.declarative_base() + SQLA model + + Returns + ------- + list of dict + Resource fields + """ + + return [ + { + "name": col.name, + "description": "", + "type": str(col.type).lower(), + "unit": "none", + } + for col in model.__table__.columns + ] + + +def generate_resource_fields_from_db_table(schema, table, geom_columns=None): + """Generate a template for the resource fields for metadata from a + database table. + + For details on the fields see field 14.6.1 of `Open Energy Metadata + `_ standard. + The fields `name` and `type` are automatically filled, the `description` + and `unit` must be filled manually. + + Examples + -------- + >>> from egon.data.metadata import generate_resource_fields_from_db_table + >>> resources = generate_resource_fields_from_db_table( + ... 'openstreetmap', 'osm_point', ['geom', 'geom_centroid'] + ... ) # doctest: +SKIP + + Parameters + ---------- + schema : str + The target table's database schema + table : str + Database table on which to put the given comment + geom_columns : list of str + Names of all geometry columns in the table. This is required to return + Geometry data type for those columns as SQL Alchemy does not recognize + them correctly. Defaults to ['geom']. + + Returns + ------- + list of dict + Resource fields + """ + + # handle geometry columns + if geom_columns is None: + geom_columns = ["geom"] + for col in geom_columns: + ischema_names[col] = Geometry + + table = Table( + table, MetaData(), schema=schema, autoload=True, autoload_with=engine() + ) + + return [ + { + "name": col.name, + "description": "", + "type": str(col.type).lower(), + "unit": "none", + } + for col in table.c + ] + + +def sources(): + shared_licenses = [license_geonutzv("© BGR, Hannover, 2021")] + shared_path = "https://dx.doi.org/10.5281/zenodo.4896526" + shared_title = ( + "Informationssystem Salz: Planungsgrundlagen, Auswahlkriterien" + " und Potenzialabschätzung für die Errichtung von Salzkavernen" + " zur Speicherung von Erneuerbaren Energien" + " (Wasserstoff und Druckluft)" + " – Doppelsalinare und flach lagernde Salzschichten." + " Teilprojekt Bewertungskriterien und Potenzialabschätzung" + ) + return { + "bgr_inspee": { + "title": "Salt structures in Northern Germany", + "description": ( + 'The application "Information System Salt Structures"' + " provides information about the areal distribution of" + " salt structures (stocks and pillows) in Northern" + " Germany. With general structural describing" + " information, such as depth, secondary thickness," + " types of use or state of exploration, queries can be" + " conducted. Contours of the salt structures can be" + " displayed at horizontal cross-sections at four" + " different depths up to a maximum depth of 2000 m" + " below NN. A data sheet with information and further" + " reading is provided for every single salt structure." + " Taking into account the fact that this work was" + " undertaken at a scale for providing an overview and" + " not for investigation of single structures, the scale" + " of display is limited to a minimum of 1:300.000." + " This web application is the product of a BMWi-funded" + ' research project "InSpEE" running from the year 2012' + ' to 2015. The acronym stands for "Information system' + " salt structures: planning basis, selection criteria" + " and estimation of the potential for the construction" + " of salt caverns for the storage of renewable energies" + ' (hydrogen and compressed air)".' + ), + "path": ( + "https://produktcenter.bgr.de/terraCatalog/DetailResult.do" + "?fileIdentifier=338136ea-261a-4569-a2bf-92999d09bad2" + ), + "licenses": [license_geonutzv("© BGR, Hannover, 2015")], + }, + "bgr_inspeeds": { + "title": "Flat layered salts in Germany", + "description": ( + "Which salt formations are suitable for storing" + " hydrogen or compressed air?" + " In the InSpEE-DS research project, scientists" + " developed requirements and criteria for the" + " assessment of suitable sites even if their" + " exploration is still at an early stage and there is" + ' little knowledge of the salinaries structures.' + " Scientists at DEEP.KBB GmbH in Hanover, worked" + " together with their project partners at the Federal" + " Institute for Geosciences and Natural Resources and" + " the Leibniz University Hanover, Institute for" + " Geotechnics Hanover, to develop the planning basis" + " for the site selection and for the construction of" + " storage caverns in flat layered salt and multiple or" + " double saliniferous formations." + " Such caverns could store renewable energy in the form" + " of hydrogen or compressed air." + " While the previous project InSpEE was limited to salt" + " formations of great thickness in Northern Germany," + " salt horizons of different ages have now been" + " examined all over Germany. To estimate the potential," + " depth contour maps of the top and the base as well as" + " thickness maps of the respective stratigraphic units" + " and reference profiles were developed. Information on" + " compressed air and hydrogen storage potential were" + " given for the identified areas and for the individual" + " federal states. The web service" + ' "Information system for flat layered salt"' + " gives access to this data. The scale of display is" + " limited to a minimum of 1:300.000. This geographic" + " information is product of a BMWi-funded research" + ' project "InSpEE-DS" running from the year 2015 to' + " 2019. The acronym stands for" + ' "Information system salt: planning basis, selection' + " criteria and estimation of the potential for the" + " construction of salt caverns for the storage of" + " renewable energies (hydrogen and compressed air)" + ' - double saline and flat salt layers".' + ), + "path": ( + "https://produktcenter.bgr.de/terraCatalog/DetailResult.do" + "?fileIdentifier=630430b8-4025-4d6f-9a62-025b53bc8b3d" + ), + "licenses": shared_licenses, + }, + "bgr_inspeeds_data_bundle": { + "title": shared_title, + "description": ( + "Shapefiles corresponding to the data provided in" + " figure 7-1 (Donadei, S., et al., 2020, p. 7-5)." + " The energy storage potential data are provided per" + " federal state in table 7-1" + " (Donadei, S., et al., 2020, p. 7-4)." + " Note: Please include all bgr data sources when using" + " the data." + ), + "path": shared_path, + "licenses": shared_licenses, + }, + "bgr_inspeeds_report": { + "title": shared_title, + "description": ( + "The report includes availability of saltstructures for" + " energy storage and energy storage potential" + " accumulated per federal state in Germany." + ), + "path": ( + "https://www.bgr.bund.de/DE/Themen" + "/Nutzung_tieferer_Untergrund_CO2Speicherung/Downloads" + "/InSpeeDS_TP_Bewertungskriterien.pdf" + "?__blob=publicationFile&v=3" + ), + "licenses": shared_licenses, + }, + "demandregio": { + "title": "DemandRegio", + "description": ( + "Harmonisierung und Entwicklung von Verfahren zur" + " regionalen und zeitlichen Auflösung von" + " Energienachfragen" + ), + "path": "https://doi.org/10.34805/ffe-119-20", + "licenses": [license_ccby("© FZJ, TUB, FfE")], + }, + "egon-data": { + "title": "eGon-data", + "description": ( + "Workflow to download, process and generate data sets" + " suitable for the further research conducted in the" + " project eGon (https://ego-n.org/)" + ), + "path": "https://github.com/openego/eGon-data", + "licenses": [license_agpl(EGON_ATTRIBUTION)], + }, + "egon-data_bundle": { + "title": "Data bundle for egon-data", + "description": ( + "Zenodo repository to provide several different input" + " data sets for eGon-data" + ), + "path": "https://zenodo.org/record/10226009", + "licenses": [license_ccby("© eGon development team")], + }, + "Einspeiseatlas": { + "title": "Einspeiseatlas", + "description": ( + "Im Einspeiseatlas finden sie sich die Informationen zu" + " realisierten und geplanten Biomethanaufbereitungsanlagen" + " - mit und ohne Einspeisung ins Gasnetz -" + " in Deutschland und weltweit." + ), + "path": "https://www.biogaspartner.de/einspeiseatlas/", + "licenses": [ + license_ccby("Deutsche Energie-Agentur (dena, 2021)") + ], + }, + "era5": { + "title": "ERA5 global reanalysis", + "description": ( + "ERA5 is the fifth generation ECMWF reanalysis for the" + " global climate and weather for the past 4 to 7" + " decades. Currently data is available from 1950, split" + " into Climate Data Store entries for 1950-1978" + " (preliminary back extension) and from 1979 onwards" + " (final release plus timely updates, this page)." + " ERA5 replaces the ERA-Interim reanalysis." + " See the online ERA5 documentation (" + "https://confluence.ecmwf.int/display/CKB" + "/ERA5%3A+data+documentation" + "#ERA5:datadocumentation-Dataupdatefrequency)" + " for more information." + ), + "path": ( + "https://confluence.ecmwf.int/display/CKB" + "/ERA5%3A+data+documentation" + "#ERA5:datadocumentation-Dataupdatefrequency" + ), + "licenses": [ + { + "name": "Licence to use Copernicus Products", + "title": "Licence to use Copernicus Products", + "path": ( + "https://cds.climate.copernicus.eu/api/v2/terms" + "/static/licence-to-use-copernicus-products.pdf" + ), + "instruction": ( + "This Licence is free of charge, worldwide," + " non-exclusive, royalty free and perpetual." + " Access to Copernicus Products is given for" + " any purpose in so far as it is lawful," + " whereas use may include, but is not limited" + " to: reproduction; distribution; communication" + " to the public; adaptation, modification and" + " combination with other data and information;" + " or any combination of the foregoing" + ), + "attribution": ( + "Copernicus Climate Change Service (C3S)" + " Climate Data Store" + ), + }, + ], + }, + "dsm-heitkoetter": { + "title": ( + "Assessment of the regionalised demand response" + " potential in Germany using an open source tool and" + " dataset" + ), + "description": ( + "With the expansion of renewable energies in Germany," + " imminent grid congestion events occur more often. One" + " approach for avoiding curtailment of renewable" + " energies is to cover excess feed-in by demand" + " response." + " As curtailment is often a local phenomenon, in this" + " work we determine the regional demand response" + " potential for the 401 German administrative districts" + " with a temporal resolution of 15 min, including" + " technical, socio-technical and economic restrictions." + ), + "path": "https://doi.org/10.1016/j.adapen.2020.100001", + "licenses": [ + license_ccby( + "© 2020 German Aerospace Center (DLR)," + " Institute of Networked Energy Systems." + ) + ], + }, + "hotmaps_industrial_sites": { + "titel": "industrial_sites_Industrial_Database", + "description": ( + "Georeferenced industrial sites of energy-intensive" + " industry sectors in EU28" + ), + "path": ( + "https://gitlab.com/hotmaps/industrial_sites" + "/industrial_sites_Industrial_Database" + ), + "licenses": [ + license_ccby("© 2016-2018: Pia Manz, Tobias Fleiter") + ], + }, + "hotmaps_scen_buildings": { + "titel": "scen_current_building_demand", + "description": ( + "Energy demand scenarios in buidlings until the year 2050" + " - current policy scenario" + ), + "path": "https://gitlab.com/hotmaps/scen_current_building_demand", + "licenses": [ + license_ccby( + "© 2016-2018: Michael Hartner" + ", Lukas Kranzl" + ", Sebastian Forthuber" + ", Sara Fritz" + ", Andreas Müller" + ) + ], + }, + "mastr": { + "title": "open-MaStR power unit registry", + "description": ( + "Raw data download Marktstammdatenregister (MaStR) data" + " using the webservice. All data from the" + " Marktstammdatenregister is included. There are" + " duplicates included. For further information read in" + " the documentation of the original data source:" + " https://www.marktstammdatenregister.de/MaStRHilfe" + "/subpages/statistik.html" + ), + "path": "https://zenodo.org/record/10480930", + "licenses": [ + licenses_datenlizenz_deutschland( + "© 2021 Bundesnetzagentur für Elektrizität, Gas," + " Telekommunikation, Post und Eisenbahnen" + ) + ], + }, + "nep2021": { + "title": ( + "Netzentwicklungsplan Strom 2035, Version 2021, erster" + " Entwurf" + ), + "description": ( + "Die vier deutschen Übertragungsnetzbetreiber zeigen" + " mit diesem ersten Entwurf des Netzentwicklungsplans" + " 2035, Version 2021, den benötigten Netzausbau für die" + " nächsten Jahre auf. Der NEP-Bericht beschreibt keine" + " konkreten Trassenverläufe von Übertragungsleitungen," + " sondern er dokumentiert den notwendigen" + " Übertragungsbedarf zwischen Netzknoten." + " Das heißt, es werden Anfangs- und Endpunkte von" + " zukünftigen Leitungsverbindungen definiert sowie" + " konkrete Empfehlungen für den Aus- und Neubau der" + " Übertragungsnetze an Land und auf See in Deutschland" + " gemäß den Detailanforderungen im § 12 EnWG gegeben." + ), + "path": "https://zenodo.org/record/5743452#.YbCoz7so8go", + "licenses": [license_ccby("© Übertragungsnetzbetreiber")], + }, + "openffe_gas": { + "title": ( + "Load Curves of the Industry Sector" + " – eXtremOS solidEU Scenario (Europe NUTS-3)" + ), + "description": ( + "Load Curves of the Industry Sector for the eXtremOS" + " solidEU Scenario Scenario at NUTS-3-Level." + " More information at https://extremos.ffe.de/." + ), + "path": ( + "http://opendata.ffe.de/dataset" + "/load-curves-of-the-industry-sector-extremos-solideu" + "-scenario-europe-nuts-3/" + ), + "licenses": [license_ccby("© FfE, eXtremOS Project")], + }, + "openstreetmap": { + "title": "OpenStreetMap Data Extracts (Geofabrik)", + "description": ( + "Full data extract of OpenStreetMap data for defined" + ' spatial extent at "referenceDate"' + ), + "path": ( + "https://download.geofabrik.de/europe/germany-210101.osm.pbf" + ), + "licenses": [license_odbl("© OpenStreetMap contributors")], + }, + "peta": { + "title": "Pan-European Thermal Atlas, Peta version 5.0.1", + "description": ( + "Modelled Heat Demand distribution (in GJ per hectare" + " grid cell) for residential and service heat demands" + " for space heating and hot water for the year 2015" + " using HRE4 data and the combined top-down bottom-up" + " approach of HRE4. National sector-specific heat" + " demand data, derived by the FORECAST model in HRE4" + " for residential (delivered energy, for space heating" + " and hot water) and service-sector (delivered energy," + " for space heating, hot water and process heat)" + " buildings for the year 2015, were distributed using" + " modelled, spatial statistics based floor areas in" + " 100x100m grids and a population grid. For further" + " information please see the documentation available on" + " the Heat Roadmap Europe website, in particular D2.3" + " report: Methodologies and assumptions used in the" + " mapping." + ), + "path": "https://s-eenergies-open-data-euf.hub.arcgis.com/search", + "licenses": [ + license_ccby( + "© Europa-Universität Flensburg" + ", Halmstad University and Aalborg University" + ) + ], + }, + "pipeline_classification": { + "title": ( + "Technical pipeline characteristics for high pressure" + " pipelines" + ), + "description": ( + "Parameters for the classification of gas pipelines," + " the whole documentation could is available at:" + " https://www.econstor.eu/bitstream/10419/173388/1" + "/1011162628.pdf" + ), + "path": "https://zenodo.org/record/5743452", + "licenses": [license_ccby("© DIW Berlin, 2017")], + }, + "schmidt": { + "title": ( + "Supplementary material to the masters thesis:" + " NUTS-3 Regionalization of Industrial Load Shifting" + " Potential in Germany using a Time-Resolved Model" + ), + "description": ( + "Supplementary material to the named masters thesis," + " containing data on industrial processes for the" + " estimation of NUTS-3 load shifting potential of" + " suitable electrically powered industrial processes" + " (cement milling, mechanical pulping, paper" + " production, air separation)." + ), + "path": "https://zenodo.org/record/3613767", + "licenses": [license_ccby("© 2019 Danielle Schmidt")], + }, + "SciGRID_gas": { + "title": "SciGRID_gas IGGIELGN", + "description": ( + "The SciGRID_gas dataset represents the European gas" + " transport network (pressure levels of 20 bars and" + " higher) including the geo-referenced transport" + " pipelines, compressor stations, LNG terminals," + " storage, production sites, gas power plants, border" + " points, and demand time series." + ), + "path": shared_path, + "licenses": [ + license_ccby( + "Jan Diettrich; Adam Pluta; Wided Medjroubi (DLR-VE)" + ), + ], + }, + "seenergies": { + "title": "D5 1 Industry Dataset With Demand Data", + "description": ( + "Georeferenced EU28 industrial sites with quantified" + " annual excess heat volumes and demand data within" + " main sectors: Chemical industry, Iron and steel," + " Non-ferrous metals, Non-metallic minerals, Paper and" + " printing, and Refineries." + ), + "path": ( + "https://s-eenergies-open-data-euf.hub.arcgis.com" + "/datasets/5e36c0af918040ed936b4e4c101f611d_0/about" + ), + "licenses": [license_ccby("© Europa-Universität Flensburg")], + }, + "technology-data": { + "titel": "Energy System Technology Data v0.3.0", + "description": ( + "This script compiles assumptions on energy system" + " technologies (such as costs, efficiencies, lifetimes," + " etc.) for chosen years (e.g. [2020, 2030, 2050]) from" + " a variety of sources into CSV files to be read by" + " energy system modelling software. The merged outputs" + " have standardized cost years, technology names, units" + " and source information." + ), + "path": "https://github.com/PyPSA/technology-data/tree/v0.3.0", + "licenses": [ + license_agpl( + "© Marta Victoria (Aarhus University)" + ", Kun Zhu (Aarhus University)" + ", Elisabeth Zeyen (TUB)" + ", Tom Brown (TUB)" + ) + ], + }, + "tyndp": { + "description": ( + "ENTSOs’ TYNDP 2020 Scenario Report describes possible" + " European energy futures up to 2050. Scenarios are not" + " forecasts; they set out a range of possible futures" + " used by the ENTSOs to test future electricity and gas" + " infrastructure needs and projects. The scenarios are" + " ambitious as they deliver a low carbon energy system" + " for Europe by 2050. The ENTSOs have developed" + " credible scenarios that are guided by technically" + " sound pathways, while reflecting country by country" + " specifics, so that a pan-European low carbon future" + " is achieved." + ), + "path": "https://tyndp.entsoe.eu/maps-data", + "licenses": [license_ccby("© ENTSO-E and ENTSOG")], + }, + "vg250": { + "title": "Verwaltungsgebiete 1:250 000 (Ebenen)", + "description": ( + "Der Datenbestand umfasst sämtliche Verwaltungseinheiten" + " der hierarchischen Verwaltungsebenen vom Staat bis zu" + " den Gemeinden mit ihren Grenzen, statistischen" + " Schlüsselzahlen, Namen der Verwaltungseinheit sowie" + " die spezifische Bezeichnung der Verwaltungsebene des" + " jeweiligen Landes." + ), + "path": ( + "https://daten.gdz.bkg.bund.de/produkte/vg" + "/vg250_ebenen_0101/2020" + "/vg250_01-01.geo84.shape.ebenen.zip" + ), + "licenses": [ + licenses_datenlizenz_deutschland( + "© Bundesamt für Kartographie und Geodäsie" + " 2020 (Daten verändert)" + ) + ], + }, + "zensus": { + "title": ( + "Statistisches Bundesamt (Destatis)" + " - Ergebnisse des Zensus 2011 zum Download" + ), + "description": ( + "Als Download bieten wir Ihnen auf dieser Seite" + " zusätzlich zur Zensusdatenbank CSV- und teilweise" + " Excel-Tabellen mit umfassenden Personen-, Haushalts-" + " und Familien- sowie Gebäude- und Wohnungs­merkmaln." + " Die Ergebnisse liegen auf Bundes-, Länder-, Kreis-" + " und Gemeinde­ebene vor. Außerdem sind einzele" + " Ergebnisse für Gitterzellen verfügbar." + ), + "path": ( + "https://www.zensus2011.de/DE/Home/Aktuelles" + "/DemografischeGrunddaten.html" + ), + "licenses": [ + licenses_datenlizenz_deutschland( + "© Statistische Ämter des Bundes und der Länder 2014" + ) + ], + }, + } + + +def contributors(authorlist): + contributors_dict = { + "am": { + "title": "Aadit Malla", + "email": "https://github.com/aadit879", + }, + "an": { + "title": "Amélia Nadal", + "email": "https://github.com/AmeliaNadal", + }, + "cb": { + "title": "Clara Büttner", + "email": "https://github.com/ClaraBuettner", + }, + "ce": { + "title": "Carlos Epia", + "email": "https://github.com/CarlosEpia", + }, + "fw": { + "title": "Francesco Witte", + "email": "https://github.com/fwitte", + }, + "gp": { + "title": "Guido Pleßmann", + "email": "https://github.com/gplssm", + }, + "ic": { + "title": "Ilka Cußmann", + "email": "https://github.com/IlkaCu", + }, + "ja": { + "title": "Jonathan Amme", + "email": "https://github.com/nesnoj", + }, + "je": { + "title": "Jane Doe", + "email": "https://github.com/JaneDoe", + }, + "ke": { + "title": "Katharina Esterl", + "email": "https://github.com/KathiEsterl", + }, + "kh": { + "title": "Kilian Helfenbein", + "email": "https://github.com/khelfen", + }, + "sg": { + "title": "Stephan Günther", + "email": "https://github.com/gnn", + }, + "um": { + "title": "Ulf Müller", + "email": "https://github.com/ulfmueller", + }, + } + return [ + {key: value for key, value in contributors_dict[author].items()} + for author in authorlist + ] + + +def upload_json_metadata(): + """Upload json metadata into db from zenodo""" + dialect = get_dialect("oep-v1.4")() + + for path in importlib_resources.files(__name__).glob("*.json"): + split = path.name.split(".") + if len(split) != 3: + continue + schema = split[0] + table = split[1] + + with open(path, "r") as infile: + obj = dialect.parse(infile.read()) + + metadata = f"'{dialect.compile_and_render(obj)}'" + db.submit_comment(metadata, schema, table) + logger.info(f"Metadata comment for {schema}.{table} stored.") + + +class Json_Metadata(Dataset): + def __init__(self, dependencies): + super().__init__( + name="JsonMetadata", + version="0.0.0", + dependencies=dependencies, + tasks={upload_json_metadata}, + ) diff --git a/src/egon/data/metadata/boundaries.egon_map_zensus_climate_zones.json b/src/egon/data/metadata/boundaries.egon_map_zensus_climate_zones.json new file mode 100644 index 000000000..f3f5a56e6 --- /dev/null +++ b/src/egon/data/metadata/boundaries.egon_map_zensus_climate_zones.json @@ -0,0 +1,151 @@ +{ + "name": "boundaries.egon_map_zensus_climate_zones", + "title": "eGon map census cells to climate zones", + "id": "WILL_BE_SET_AT_PUBLICATION", + "description": "spatial join of census cells and climate zones", + "language": [ + "en-EN", + "de-DE" + ], + "keywords": [], + "publicationDate": "2022-09-14", + "context": { + "homepage": "https://ego-n.org/", + "documentation": "https://egon-data.readthedocs.io/en/latest/", + "sourceCode": "https://github.com/openego/eGon-data", + "contact": "https://ego-n.org/partners/", + "grantNo": "03EI1002", + "fundingAgency": "Bundesministerium f\u00fcr Wirtschaft und Klimaschutz", + "fundingAgencyLogo": "https://www.innovation-beratung-foerderung.de/INNO/Redaktion/DE/Bilder/Titelbilder/titel_foerderlogo_bmwi.jpg?__blob=normal&v=3", + "publisherLogo": "https://ego-n.org/images/eGon_logo_noborder_transbg.svg" + }, + "spatial": { + "location": "", + "extent": "Germany", + "resolution": "100 m x 100 m" + }, + "temporal": { + "referenceDate": "2022-09-14", + "timeseries": {} + }, + "sources": [ + { + "title": "eGon-data", + "description": "Workflow to download, process and generate data sets suitable for the further research conducted in the project eGon (https://ego-n.org/)", + "path": "https://github.com/openego/eGon-data", + "licenses": [ + { + "name": "AGPL-3.0 License", + "title": "GNU Affero General Public License v3.0", + "path": "https://www.gnu.org/licenses/agpl-3.0.de.html", + "instruction": "Permissions of this strongest copyleft license are conditioned on making available complete source code of licensed works and modifications, which include larger works using a licensed work, under the same license. Copyright and license notices must be preserved. Contributors provide an express grant of patent rights. When a modified version is used to provide a service over a network, the complete source code of the modified version must be made available.", + "attribution": "\u00a9 eGon-data developers" + } + ] + }, + { + "title": "Data bundle for egon-data: A transparent and reproducible data processing pipeline for energy system modeling", + "description": "egon-data provides a transparent and reproducible open data based data processing pipeline for generating data models suitable for energy system modeling. The data is customized for the requirements of the research project eGon. The research project aims to develop tools for an open and cross-sectoral planning of transmission and distribution grids. For further information please visit the eGon project website or its Github repository. egon-data retrieves and processes data from several different external input sources. As not all data dependencies can be downloaded automatically from external sources we provide a data bundle to be downloaded by egon-data.", + "path": "https://doi.org/10.5281/zenodo.6630616", + "licenses": [ + { + "name": "CC-BY-4.0", + "title": "Creative Commons Attribution 4.0 International", + "path": "https://creativecommons.org/licenses/by/4.0/legalcode", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute.", + "attribution": "\u00a9 eGon-data developers, \u00a9 Leibniz Institute for Applied Geophysics, \u00a9 Fraunhofer IEE, \u00a9 BGR Hannover, \u00a9 Danielle Schmidt, \u00a9 \u00dcbertragungsnetzbetreiber, \u00a9 DIW Berlin, \u00a9 Reiner Lemoine Institut, \u00a9 Statistisches Bundesamt, Wiesbaden 2008, \u00a9 Statistische \u00c4mter des Bundes und der L\u00e4nder 2021" + } + ] + }, + { + "title": "Statistisches Bundesamt (Destatis) - Ergebnisse des Zensus 2011 zum Download", + "description": "Als Download bieten wir Ihnen auf dieser Seite zus\u00e4tzlich zur Zensusdatenbank CSV- und eilweise Excel-Tabellen mit umfassenden Personen-, Haushalts- und Familien- sowie Geb\u00e4ude- und Wohnungsmerkmalen. Die Ergebnisse liegen auf Bundes-, L\u00e4nder-, Kreis- und Gemeindeebene vor. Au\u00dferdem sind einzelne Ergebnisse f\u00fcr Gitterzellen verf\u00fcgbar.", + "path": "https://www.zensus2011.de/DE/Home/Aktuelles/DemografischeGrunddaten.html", + "licenses": [ + { + "name": "dl-by-de/2.0", + "title": "Datenlizenz Deutschland \u2013 Namensnennung \u2013 Version 2.0", + "path": "www.govdata.de/dl-de/by-2-0", + "instruction": "Jede Nutzung ist unter den Bedingungen dieser \u201eDatenlizenz Deutschland - Namensnennung - Version 2.0 zul\u00e4ssig. Die bereitgestellten Daten und Metadaten d\u00fcrfen f\u00fcr die kommerzielle und nicht kommerzielle Nutzung insbesondere: (1) vervielf\u00e4ltigt, ausgedruckt, pr\u00e4sentiert, ver\u00e4ndert, bearbeitet sowie an Dritte \u00fcbermittelt werden; (2) mit eigenen Daten und Daten Anderer zusammengef\u00fchrt und zu selbst\u00e4ndigen neuen Datens\u00e4tzen verbunden werden; (3) in interne und externe Gesch\u00e4ftsprozesse, Produkte und Anwendungen in \u00f6ffentlichen und nicht \u00f6ffentlichen elektronischen Netzwerken eingebunden werden. Bei der Nutzung ist sicherzustellen, dass folgende Angaben als Quellenvermerk enthalten sind: (1) Bezeichnung des Bereitstellers nach dessen Ma\u00dfgabe, der Vermerk Datenlizenz Deutschland \u2013 Namensnennung \u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 sowie (3) einen Verweis auf den Datensatz (URI). Dies gilt nur soweit die datenhaltende Stelle die Angaben (1) bis (3) zum Quellenvermerk bereitstellt. Ver\u00e4nderungen, Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk mit dem Hinweis zu versehen, dass die Daten ge\u00e4ndert wurden.", + "attribution": "\u00a9 Statistische \u00c4mter des Bundes und der L\u00e4nder 2014" + } + ] + } + ], + "licenses": [ + { + "name": "CC-BY-4.0", + "title": "Creative Commons Attribution 4.0 International", + "path": "https://creativecommons.org/licenses/by/4.0/legalcode", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute.", + "attribution": "\u00a9 eGon-data developers" + } + ], + "contributors": [ + { + "title": "ClaraBuettner", + "email": "https://github.com/ClaraBuettner", + "date": "2022-09-12", + "object": "metadata", + "comment": "Create metadata" + }, + { + "title": "nesnoj", + "email": "https://github.com/nesnoj", + "date": "2022-09-14", + "object": "metadata", + "comment": "Update metadata" + } + ], + "resources": [ + { + "profile": "tabular-data-resource", + "name": "boundaries.egon_map_zensus_climate_zones", + "path": "", + "format": "PostgreSQL", + "encoding": "UTF-8", + "schema": { + "fields": [ + { + "name": "climate_zone", + "description": "Name of the corresponding climate zone", + "type": "str", + "unit": "none" + }, + { + "name": "zensus_population_id", + "description": "Index of corresponding census cell (reference to: society.destatis_zensus_population_per_ha.id)", + "type": "integer", + "unit": "none" + } + ], + "primaryKey": "zensus_population_id" + }, + "dialect": { + "delimiter": "", + "decimalSeparator": "" + } + } + ], + "review": { + "path": "", + "badge": "" + }, + "metaMetadata": { + "metadataVersion": "OEP-1.4.1", + "metadataLicense": { + "name": "CC0-1.0", + "title": "Creative Commons Zero v1.0 Universal", + "path": "https://creativecommons.org/publicdomain/zero/1.0/" + } + }, + "_comment": { + "metadata": "Metadata documentation and explanation (https://github.com/OpenEnergyPlatform/oemetadata/blob/master/metadata/v141/metadata_key_description.md)", + "dates": "Dates and time must follow the ISO8601 including time zone (YYYY-MM-DD or YYYY-MM-DDThh:mm:ss\u00b1hh)", + "units": "Use a space between numbers and units (100 m)", + "languages": "Languages must follow the IETF (BCP47) format (en-GB, en-US, de-DE)", + "licenses": "License name must follow the SPDX License List (https://spdx.org/licenses/)", + "review": "Following the OEP Data Review (https://github.com/OpenEnergyPlatform/data-preprocessing/wiki)", + "none": "If not applicable use (none)" + } +} diff --git a/src/egon/data/metadata/boundaries.egon_map_zensus_mvgd_buildings.json b/src/egon/data/metadata/boundaries.egon_map_zensus_mvgd_buildings.json new file mode 100644 index 000000000..e09cf47aa --- /dev/null +++ b/src/egon/data/metadata/boundaries.egon_map_zensus_mvgd_buildings.json @@ -0,0 +1,171 @@ +{ + "name": "boundaries.egon_map_zensus_mvgd_buildings", + "title": "eGon buildings mapping table", + "id": "WILL_BE_SET_AT_PUBLICATION", + "description": "Mapping table for all used buildings via project internal building_id. Additional info about if it is a synthetic or a openstreetmap building, where it is located (census cell_id, mvgd_id), in which sector (residential, cts) the building is used, if there are heat or electricity profiles.", + "language": "undefined", + "keywords": [], + "publicationDate": "2023-03-13", + "context": { + "homepage": "https://ego-n.org/", + "documentation": "https://egon-data.readthedocs.io/en/latest/", + "sourceCode": "https://github.com/openego/eGon-data", + "contact": "https://ego-n.org/partners/", + "grantNo": "03EI1002", + "fundingAgency": "Bundesministerium f\u00fcr Wirtschaft und Klimaschutz", + "fundingAgencyLogo": "https://commons.wikimedia.org/wiki/File:BMWi_Logo_2021.svg#/media/File:BMWi_Logo_2021.svg", + "publisherLogo": "https://ego-n.org/images/eGon_logo_noborder_transbg.svg" + }, + "spatial": { + "location": "", + "extent": "Germany", + "resolution": "Buildings" + }, + "temporal": { + "referenceDate": "2023-03-13", + "timeseries": {} + }, + "sources": [ + { + "title": "Data bundle for egon-data: A transparent and reproducible data processing pipeline for energy system modeling", + "description": "egon-data provides a transparent and reproducible open data based data processing pipeline for generating data models suitable for energy system modeling. The data is customized for the requirements of the research project eGon. The research project aims to develop tools for an open and cross-sectoral planning of transmission and distribution grids. For further information please visit the eGon project website or its Github repository. egon-data retrieves and processes data from several different external input sources. As not all data dependencies can be downloaded automatically from external sources we provide a data bundle to be downloaded by egon-data.", + "path": "https://doi.org/10.5281/zenodo.6630616", + "licenses": [ + { + "name": "CC-BY-4.0", + "title": "Creative Commons Attribution 4.0 International", + "path": "https://creativecommons.org/licenses/by/4.0/legalcode", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute.", + "attribution": "\u00a9 eGon-data developers, \u00a9 Leibniz Institute for Applied Geophysics, \u00a9 Fraunhofer IEE, \u00a9 BGR Hannover, \u00a9 Danielle Schmidt, \u00a9 \u00dcbertragungsnetzbetreiber, \u00a9 DIW Berlin, \u00a9 Reiner Lemoine Institut, \u00a9 Statistisches Bundesamt, Wiesbaden 2008, \u00a9 Statistische \u00c4mter des Bundes und der L\u00e4nder 2021" + } + ] + }, + { + "title": "eGon-data", + "description": "Workflow to download, process and generate data sets suitable for the further research conducted in the project eGon (https://ego-n.org/)", + "path": "https://github.com/openego/eGon-data", + "licenses": [ + { + "name": "AGPL-3.0 License", + "title": "GNU Affero General Public License v3.0", + "path": "https://www.gnu.org/licenses/agpl-3.0.de.html", + "instruction": "Permissions of this strongest copyleft license are conditioned on making available complete source code of licensed works and modifications, which include larger works using a licensed work, under the same license. Copyright and license notices must be preserved. Contributors provide an express grant of patent rights. When a modified version is used to provide a service over a network, the complete source code of the modified version must be made available.", + "attribution": "\u00a9 eGon-data developers" + } + ] + }, + { + "title": "OpenStreetMap Data Extracts (Geofabrik)", + "description": "Full data extract of OpenStreetMap data for defined spatial extent at 2022-01-01", + "path": "https://download.geofabrik.de/europe/germany-210101.osm.pbf", + "licenses": [ + { + "name": "ODbL-1.0", + "title": "Open Data Commons Open Database License 1.0", + "path": "https://opendatacommons.org/licenses/odbl/1.0/", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute, Share-Alike, Keep open!", + "attribution": "\u00a9 OpenStreetMap contributors" + } + ] + } + ], + "licenses": [ + { + "name": "ODbL-1.0", + "title": "Open Data Commons Open Database License 1.0", + "path": "https://opendatacommons.org/licenses/odbl/1.0/", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute, Share-Alike, Keep open!", + "attribution": "\u00a9 Reiner Lemoine Institut" + } + ], + "contributors": [ + { + "title": "nailend", + "email": "https://github.com/nailend", + "date": "2023-03-13", + "object": "metadata", + "comment": "create metadata" + } + ], + "resources": [ + { + "profile": "tabular-data-resource", + "name": "boundaries.egon_map_zensus_mvgd_buildings", + "path": "", + "format": "PostgreSQL", + "encoding": "UTF-8", + "schema": { + "fields": [ + { + "name": "building_id", + "description": "Building id (reference to: openstreetmap.osm_buildings_filtered.id, openstreetmap.osm_buildings_synthetic.id)", + "type": "integer", + "unit": "none" + }, + { + "name": "sector", + "description": "Sector (cts, residential)", + "type": "characters varying", + "unit": "none" + }, + { + "name": "zensus_population_id", + "description": "Zensus population id (reference to: society.destatis_zensus_population_per_ha.id)", + "type": "integer", + "unit": "none" + }, + { + "name": "bus_id", + "description": "MVGD bus id (reference to: grid.egon_hvmv_substation.bus_id, grid.egon_mv_grid_district.bus_id)", + "type": "integer", + "unit": "none" + }, + { + "name": "electricity", + "description": "True if respectives electricity demand profile exists", + "type": "bool", + "unit": "None" + }, + { + "name": "heat", + "description": "True if respectives heat demand profile exists", + "type": "bool", + "unit": "None" + }, + { + "name": "osm", + "description": "is OSM building if true else syntheticaly created", + "type": "bool", + "unit": "None" + } + ], + "primaryKey": "building_id, sector" + }, + "dialect": { + "delimiter": "", + "decimalSeparator": "." + } + } + ], + "review": { + "path": "", + "badge": "" + }, + "metaMetadata": { + "metadataVersion": "OEP-1.4.1", + "metadataLicense": { + "name": "CC0-1.0", + "title": "Creative Commons Zero v1.0 Universal", + "path": "https://creativecommons.org/publicdomain/zero/1.0/" + } + }, + "_comment": { + "metadata": "Metadata documentation and explanation (https://github.com/OpenEnergyPlatform/oemetadata/blob/master/metadata/v141/metadata_key_description.md)", + "dates": "Dates and time must follow the ISO8601 including time zone (YYYY-MM-DD or YYYY-MM-DDThh:mm:ss\u00b1hh)", + "units": "Use a space between numbers and units (100 m)", + "languages": "Languages must follow the IETF (BCP47) format (en-GB, en-US, de-DE)", + "licenses": "License name must follow the SPDX License List (https://spdx.org/licenses/)", + "review": "Following the OEP Data Review (https://github.com/OpenEnergyPlatform/data-preprocessing/wiki)", + "none": "If not applicable use (none)" + } +} diff --git a/src/egon/data/metadata/boundaries.egon_map_zensus_weather_cell.json b/src/egon/data/metadata/boundaries.egon_map_zensus_weather_cell.json new file mode 100644 index 000000000..3026115aa --- /dev/null +++ b/src/egon/data/metadata/boundaries.egon_map_zensus_weather_cell.json @@ -0,0 +1,140 @@ +{ + "name": "boundaries.egon_map_zensus_climate_zones", + "title": "Map census cells to era5 climate zones", + "id": "WILL_BE_SET_AT_PUBLICATION", + "description": "Mapping table of census cells to era5 climate zones", + "language": "en-US", + "keywords": [ + "" + ], + "publicationDate": "2022-08-06", + "context": { + "homepage": "https://ego-n.org/", + "documentation": "https://egon-data.readthedocs.io/en/latest/", + "sourceCode": "https://github.com/openego/eGon-data", + "contact": "https://ego-n.org/partners/", + "grantNo": "03EI1002", + "fundingAgency": "Bundesministerium f\\u00fcr Wirtschaft und Energie", + "fundingAgencyLogo": "https://www.innovation-beratung-foerderung.de/INNO/Redaktion/DE/Bilder/Titelbilder/titel_foerderlogo_bmwi.jpg?__blob=normal&v=3", + "publisherLogo": "https://ego-n.org/images/eGon_logo_noborder_transbg.svg" + }, + "spatial": { + "location": "", + "extent": "Germany", + "resolution": "1 ha" + }, + "temporal": {}, + "sources": [ + { + "title": "Statistisches Bundesamt (Destatis) - Ergebnisse des Zensus 2011 zum Download", + "description": "Als Download bieten wir Ihnen auf dieser Seite zus\\u00e4tzlich zur Zensusdatenbank CSV- und teilweise Excel-Tabellen mit umfassenden Personen-, Haushalts- und Familien- sowie Geb\\u00e4ude- und Wohnungsmerkmalen. Die Ergebnisse liegen auf Bundes-, L\\u00e4nder-, Kreis- und Gemeindeebene vor. Au\\u00dferdem sind einzelne Ergebnisse f\\u00fcr Gitterzellen verf\\u00fcgbar.", + "path": "https://www.zensus2011.de/DE/Home/Aktuelles/DemografischeGrunddaten.html", + "licenses": [ + { + "name": "dl-by-de/2.0", + "title": "Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0", + "path": "www.govdata.de/dl-de/by-2-0", + "instruction": "Jede Nutzung ist unter den Bedingungen dieser \\u201eDatenlizenz Deutschland - Namensnennung - Version 2.0 zul\\u00e4ssig.\\nDie bereitgestellten Daten und Metadaten d\\u00fcrfen f\\u00fcr die kommerzielle und nicht kommerzielle Nutzung insbesondere:(1) vervielf\\u00e4ltigt, ausgedruckt, pr\\u00e4sentiert, ver\\u00e4ndert, bearbeitet sowie an Dritte \\u00fcbermittelt werden;\\n (2) mit eigenen Daten und Daten Anderer zusammengef\\u00fchrt und zu selbst\\u00e4ndigen neuen Datens\\u00e4tzen verbunden werden;\\n (3) in interne und externe Gesch\\u00e4ftsprozesse, Produkte und Anwendungen in \\u00f6ffentlichen und nicht \\u00f6ffentlichen elektronischen Netzwerken eingebunden werden.\\nBei der Nutzung ist sicherzustellen, dass folgende Angaben als Quellenvermerk enthalten sind:\\n(1) Bezeichnung des Bereitstellers nach dessen Ma\\u00dfgabe,\\n(2) der Vermerk Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 sowie\\n(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.\\nVer\\u00e4nderungen, Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk mit dem Hinweis zu versehen, dass die Daten ge\\u00e4ndert wurden.", + "attribution": "\\u00a9 Statistische \\u00c4mter des Bundes und der L\\u00e4nder 2014" + } + ] + }, + { + "title": "Dokumentation - Zensus 2011 - Methoden und Verfahren", + "description": "Diese Publikation beschreibt ausf\\u00fchrlich die Methoden und Verfahren des registergest\\u00fctzten Zensus 2011; von der Datengewinnung und -aufbereitung bis hin zur Ergebniserstellung und Geheimhaltung. Der vorliegende Band wurde von den Statistischen \\u00c4mtern des Bundes und der L\\u00e4nder im Juni 2015 ver\\u00f6ffentlicht.", + "path": "https://www.destatis.de/DE/Publikationen/Thematisch/Bevoelkerung/Zensus/ZensusBuLaMethodenVerfahren5121105119004.pdf?__blob=publicationFile", + "licenses": [ + { + "name": "dl-by-de/2.0", + "title": "Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0", + "path": "www.govdata.de/dl-de/by-2-0", + "instruction": "Jede Nutzung ist unter den Bedingungen dieser \\u201eDatenlizenz Deutschland - Namensnennung - Version 2.0 zul\\u00e4ssig.\\nDie bereitgestellten Daten und Metadaten d\\u00fcrfen f\\u00fcr die kommerzielle und nicht kommerzielle Nutzung insbesondere:(1) vervielf\\u00e4ltigt, ausgedruckt, pr\\u00e4sentiert, ver\\u00e4ndert, bearbeitet sowie an Dritte \\u00fcbermittelt werden;\\n (2) mit eigenen Daten und Daten Anderer zusammengef\\u00fchrt und zu selbst\\u00e4ndigen neuen Datens\\u00e4tzen verbunden werden;\\n (3) in interne und externe Gesch\\u00e4ftsprozesse, Produkte und Anwendungen in \\u00f6ffentlichen und nicht \\u00f6ffentlichen elektronischen Netzwerken eingebunden werden.\\nBei der Nutzung ist sicherzustellen, dass folgende Angaben als Quellenvermerk enthalten sind:\\n(1) Bezeichnung des Bereitstellers nach dessen Ma\\u00dfgabe,\\n(2) der Vermerk Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 sowie\\n(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.\\nVer\\u00e4nderungen, Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk mit dem Hinweis zu versehen, dass die Daten ge\\u00e4ndert wurden.", + "attribution": "\\u00a9 Statistisches Bundesamt, Wiesbaden 2015 (im Auftrag der Herausgebergemeinschaft)" + } + ] + }, + { + "title": "ERA5 global reanalysis", + "description": "ERA5 is the fifth generation ECMWF reanalysis for the global climate and weather for the past 4 to 7 decades. Currently data is available from 1950, split into Climate Data Store entries for 1950-1978 (preliminary back extension) and from 1979 onwards (final release plus timely updates, this page). ERA5 replaces the ERA-Interim reanalysis. See the online ERA5 documentation (https://confluence.ecmwf.int/display/CKB/ERA5%3A+data+documentation#ERA5:datadocumentation-Dataupdatefrequency) for more information.", + "path": "https://confluence.ecmwf.int/display/CKB/ERA5%3A+data+documentation#ERA5:datadocumentation-Dataupdatefrequency", + "licenses": [ + { + "name": "Licence to use Copernicus Products", + "title": "Licence to use Copernicus Products", + "path": "https://cds.climate.copernicus.eu/api/v2/terms/static/licence-to-use-copernicus-products.pdf", + "instruction": "This Licence is free of charge, worldwide, non-exclusive, royalty free and perpetual. Access to Copernicus Products is given for any purpose in so far as it is lawful, whereas use may include, but is not limited to: reproduction; distribution; communication to the public; adaptation, modification and combination with other data and information; or any combination of the foregoing", + "attribution": "\u00a9 Copernicus Climate Change Service (C3S) Climate Data Store" + } + ] + } + ], + "licenses": [ + { + "name": "Creative Commons Attribution 4.0 International", + "title": "CC BY 4.0", + "path": "https://creativecommons.org/licenses/by/4.0/", + "instruction": "You are free: To Share, To Adapt; As long as you: Attribute!", + "attribution": "\\u00a9 eGon development team" + } + ], + "contributors": [ + { + "title": "nailend", + "email": "https://github.com/nailend", + "date": "2023-03-13", + "object": "metadata", + "comment": "Create metadata" + } + ], + "resources": [ + { + "profile": "tabular-data-resource", + "name": "boundaries.egon_map_zensus_weather_cell", + "path": "", + "format": "PostgreSQL", + "encoding": "UTF-8", + "schema": { + "fields": [ + { + "name": "zensus_population_id", + "description": "Zensus population id (reference to: society.destatis_zensus_population_per_ha.id)", + "type": "integer", + "unit": "none" + }, + { + "name": "w_id", + "description": "Weather cell id (reference to: supply.egon_era5_weather_cells)", + "type": "Integer", + "unit": "none" + } + ], + "primaryKey": "zensus_population_id" + }, + "dialect": { + "delimiter": "", + "decimalSeparator": "." + } + } + ], + "review": { + "path": "", + "badge": "" + }, + "metaMetadata": { + "metadataVersion": "OEP-1.4.1", + "metadataLicense": { + "name": "CC0-1.0", + "title": "Creative Commons Zero v1.0 Universal", + "path": "https://creativecommons.org/publicdomain/zero/1.0/" + } + }, + "_comment": { + "metadata": "Metadata documentation and explanation (https://github.com/OpenEnergyPlatform/oemetadata/blob/master/metadata/v141/metadata_key_description.md)", + "dates": "Dates and time must follow the ISO8601 including time zone (YYYY-MM-DD or YYYY-MM-DDThh:mm:ss\u00b1hh)", + "units": "Use a space between numbers and units (100 m)", + "languages": "Languages must follow the IETF (BCP47) format (en-GB, en-US, de-DE)", + "licenses": "License name must follow the SPDX License List (https://spdx.org/licenses/)", + "review": "Following the OEP Data Review (https://github.com/OpenEnergyPlatform/data-preprocessing/wiki)", + "none": "If not applicable use (none)" + } +} diff --git a/src/egon/data/metadata/demand.egon_building_electricity_peak_loads.json b/src/egon/data/metadata/demand.egon_building_electricity_peak_loads.json new file mode 100644 index 000000000..19d7b55cc --- /dev/null +++ b/src/egon/data/metadata/demand.egon_building_electricity_peak_loads.json @@ -0,0 +1,201 @@ +{ + "name": "demand.egon_building_electricity_peak_loads", + "title": "Electricity peak loads for egon buildings", + "id": "WILL_BE_SET_AT_PUBLICATION", + "description": "Electricity peak load table of residential and cts buildings with voltage level for two scenarios ", + "language": "undefined", + "keywords": [], + "publicationDate": "2023-03-13", + "context": { + "homepage": "https://ego-n.org/", + "documentation": "https://egon-data.readthedocs.io/en/latest/", + "sourceCode": "https://github.com/openego/eGon-data", + "contact": "https://ego-n.org/partners/", + "grantNo": "03EI1002", + "fundingAgency": "Bundesministerium f\u00fcr Wirtschaft und Klimaschutz", + "fundingAgencyLogo": "https://commons.wikimedia.org/wiki/File:BMWi_Logo_2021.svg#/media/File:BMWi_Logo_2021.svg", + "publisherLogo": "https://ego-n.org/images/eGon_logo_noborder_transbg.svg" + }, + "spatial": { + "location": "", + "extent": "Germany", + "resolution": "Buildings" + }, + "temporal": { + "referenceDate": "2023-03-13", + "timeseries": {} + }, + "sources": [ + { + "title": "Data bundle for egon-data: A transparent and reproducible data processing pipeline for energy system modeling", + "description": "egon-data provides a transparent and reproducible open data based data processing pipeline for generating data models suitable for energy system modeling. The data is customized for the requirements of the research project eGon. The research project aims to develop tools for an open and cross-sectoral planning of transmission and distribution grids. For further information please visit the eGon project website or its Github repository. egon-data retrieves and processes data from several different external input sources. As not all data dependencies can be downloaded automatically from external sources we provide a data bundle to be downloaded by egon-data.", + "path": "https://doi.org/10.5281/zenodo.6630616", + "licenses": [ + { + "name": "CC-BY-4.0", + "title": "Creative Commons Attribution 4.0 International", + "path": "https://creativecommons.org/licenses/by/4.0/legalcode", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute.", + "attribution": "\u00a9 eGon-data developers, \u00a9 Leibniz Institute for Applied Geophysics, \u00a9 Fraunhofer IEE, \u00a9 BGR Hannover, \u00a9 Danielle Schmidt, \u00a9 \u00dcbertragungsnetzbetreiber, \u00a9 DIW Berlin, \u00a9 Reiner Lemoine Institut, \u00a9 Statistisches Bundesamt, Wiesbaden 2008, \u00a9 Statistische \u00c4mter des Bundes und der L\u00e4nder 2021" + } + ] + }, + { + "title": "eGon-data", + "description": "Workflow to download, process and generate data sets suitable for the further research conducted in the project eGon (https://ego-n.org/)", + "path": "https://github.com/openego/eGon-data", + "licenses": [ + { + "name": "AGPL-3.0 License", + "title": "GNU Affero General Public License v3.0", + "path": "https://www.gnu.org/licenses/agpl-3.0.de.html", + "instruction": "Permissions of this strongest copyleft license are conditioned on making available complete source code of licensed works and modifications, which include larger works using a licensed work, under the same license. Copyright and license notices must be preserved. Contributors provide an express grant of patent rights. When a modified version is used to provide a service over a network, the complete source code of the modified version must be made available.", + "attribution": "\u00a9 eGon-data developers" + } + ] + }, + { + "title": "Statistisches Bundesamt (Destatis) - Ergebnisse des Zensus 2011 zum Download", + "description": "Als Download bieten wir Ihnen auf dieser Seite zus\\u00e4tzlich zur Zensusdatenbank CSV- und teilweise Excel-Tabellen mit umfassenden Personen-, Haushalts- und Familien- sowie Geb\\u00e4ude- und Wohnungsmerkmalen. Die Ergebnisse liegen auf Bundes-, L\\u00e4nder-, Kreis- und Gemeindeebene vor. Au\\u00dferdem sind einzelne Ergebnisse f\\u00fcr Gitterzellen verf\\u00fcgbar.", + "path": "https://www.zensus2011.de/DE/Home/Aktuelles/DemografischeGrunddaten.html", + "licenses": [ + { + "name": "dl-by-de/2.0", + "title": "Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0", + "path": "www.govdata.de/dl-de/by-2-0", + "instruction": "Jede Nutzung ist unter den Bedingungen dieser \\u201eDatenlizenz Deutschland - Namensnennung - Version 2.0 zul\\u00e4ssig.\\nDie bereitgestellten Daten und Metadaten d\\u00fcrfen f\\u00fcr die kommerzielle und nicht kommerzielle Nutzung insbesondere:(1) vervielf\\u00e4ltigt, ausgedruckt, pr\\u00e4sentiert, ver\\u00e4ndert, bearbeitet sowie an Dritte \\u00fcbermittelt werden;\\n (2) mit eigenen Daten und Daten Anderer zusammengef\\u00fchrt und zu selbst\\u00e4ndigen neuen Datens\\u00e4tzen verbunden werden;\\n (3) in interne und externe Gesch\\u00e4ftsprozesse, Produkte und Anwendungen in \\u00f6ffentlichen und nicht \\u00f6ffentlichen elektronischen Netzwerken eingebunden werden.\\nBei der Nutzung ist sicherzustellen, dass folgende Angaben als Quellenvermerk enthalten sind:\\n(1) Bezeichnung des Bereitstellers nach dessen Ma\\u00dfgabe,\\n(2) der Vermerk Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 sowie\\n(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.\\nVer\\u00e4nderungen, Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk mit dem Hinweis zu versehen, dass die Daten ge\\u00e4ndert wurden.", + "attribution": "\\u00a9 Statistische \\u00c4mter des Bundes und der L\\u00e4nder 2014" + } + ] + }, + { + "title": "Dokumentation - Zensus 2011 - Methoden und Verfahren", + "description": "Diese Publikation beschreibt ausf\\u00fchrlich die Methoden und Verfahren des registergest\\u00fctzten Zensus 2011; von der Datengewinnung und -aufbereitung bis hin zur Ergebniserstellung und Geheimhaltung. Der vorliegende Band wurde von den Statistischen \\u00c4mtern des Bundes und der L\\u00e4nder im Juni 2015 ver\\u00f6ffentlicht.", + "path": "https://www.destatis.de/DE/Publikationen/Thematisch/Bevoelkerung/Zensus/ZensusBuLaMethodenVerfahren5121105119004.pdf?__blob=publicationFile", + "licenses": [ + { + "name": "dl-by-de/2.0", + "title": "Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0", + "path": "www.govdata.de/dl-de/by-2-0", + "instruction": "Jede Nutzung ist unter den Bedingungen dieser \\u201eDatenlizenz Deutschland - Namensnennung - Version 2.0 zul\\u00e4ssig.\\nDie bereitgestellten Daten und Metadaten d\\u00fcrfen f\\u00fcr die kommerzielle und nicht kommerzielle Nutzung insbesondere:(1) vervielf\\u00e4ltigt, ausgedruckt, pr\\u00e4sentiert, ver\\u00e4ndert, bearbeitet sowie an Dritte \\u00fcbermittelt werden;\\n (2) mit eigenen Daten und Daten Anderer zusammengef\\u00fchrt und zu selbst\\u00e4ndigen neuen Datens\\u00e4tzen verbunden werden;\\n (3) in interne und externe Gesch\\u00e4ftsprozesse, Produkte und Anwendungen in \\u00f6ffentlichen und nicht \\u00f6ffentlichen elektronischen Netzwerken eingebunden werden.\\nBei der Nutzung ist sicherzustellen, dass folgende Angaben als Quellenvermerk enthalten sind:\\n(1) Bezeichnung des Bereitstellers nach dessen Ma\\u00dfgabe,\\n(2) der Vermerk Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 sowie\\n(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.\\nVer\\u00e4nderungen, Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk mit dem Hinweis zu versehen, dass die Daten ge\\u00e4ndert wurden.", + "attribution": "\\u00a9 Statistisches Bundesamt, Wiesbaden 2015 (im Auftrag der Herausgebergemeinschaft)" + } + ] + }, + { + "title": "OpenStreetMap Data Extracts (Geofabrik)", + "description": "Full data extract of OpenStreetMap data for defined spatial extent at 2022-01-01", + "path": "https://download.geofabrik.de/europe/germany-210101.osm.pbf", + "licenses": [ + { + "name": "ODbL-1.0", + "title": "Open Data Commons Open Database License 1.0", + "path": "https://opendatacommons.org/licenses/odbl/1.0/", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute, Share-Alike, Keep open!", + "attribution": "\u00a9 OpenStreetMap contributors" + } + ] + }, + { + "title": "DemandRegio", + "description": "Harmonisierung und Entwicklung von Verfahren zur regionalen und zeitlichen Aufl\u00f6sung von Energienachfragen", + "path": "https://doi.org/10.34805/ffe-119-20", + "license": [ + { + "name": "Creative Commons Attribution 4.0 International", + "title": "CC BY 4.0", + "path": "https://creativecommons.org/licenses/by/4.0/", + "instruction": "You are free: To Share, To Adapt; As long as you: Attribute!", + "attribution": "\u00a9 FZJ, TUB, FfE" + } + ] + } + ], + "licenses": [ + { + "name": "ODbL-1.0", + "title": "Open Data Commons Open Database License 1.0", + "path": "https://opendatacommons.org/licenses/odbl/1.0/", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute, Share-Alike, Keep open!", + "attribution": "\u00a9 Reiner Lemoine Institut" + } + ], + "contributors": [ + { + "title": "nailend", + "email": "https://github.com/nailend", + "date": "2023-03-13", + "object": "metadata", + "comment": "create metadata" + } + ], + "resources": [ + { + "profile": "tabular-data-resource", + "name": "demand.egon_building_electricity_peak_loads", + "path": "", + "format": "PostgreSQL", + "encoding": "UTF-8", + "schema": { + "fields": [ + { + "name": "building_id", + "description": "Building id (reference to: openstreetmap.osm_buildings_filtered.id, openstreetmap.osm_buildings_synthetic.id, boundaries.egon_map_zensus_mvgd_building)", + "type": "integer", + "unit": "none" + }, + { + "name": "scenario", + "description": "Scenario (eGon100RE, eGon2035)", + "type": "character varying", + "unit": "none" + }, + { + "name": "sector", + "description": "Sector (cts, residential)", + "type": "characters varying", + "unit": "none" + }, + { + "name": "peak_load_in_w", + "description": "Peak load value", + "type": "real", + "unit": "W" + }, + { + "name": "voltage_level", + "description": "Voltage level of building connection after MitnetzStrom Classification", + "type": "int", + "unit": "None" + } + ], + "primaryKey": "building_id, scenario, sector" + }, + "dialect": { + "delimiter": "", + "decimalSeparator": "." + } + } + ], + "review": { + "path": "", + "badge": "" + }, + "metaMetadata": { + "metadataVersion": "OEP-1.4.1", + "metadataLicense": { + "name": "CC0-1.0", + "title": "Creative Commons Zero v1.0 Universal", + "path": "https://creativecommons.org/publicdomain/zero/1.0/" + } + }, + "_comment": { + "metadata": "Metadata documentation and explanation (https://github.com/OpenEnergyPlatform/oemetadata/blob/master/metadata/v141/metadata_key_description.md)", + "dates": "Dates and time must follow the ISO8601 including time zone (YYYY-MM-DD or YYYY-MM-DDThh:mm:ss\u00b1hh)", + "units": "Use a space between numbers and units (100 m)", + "languages": "Languages must follow the IETF (BCP47) format (en-GB, en-US, de-DE)", + "licenses": "License name must follow the SPDX License List (https://spdx.org/licenses/)", + "review": "Following the OEP Data Review (https://github.com/OpenEnergyPlatform/data-preprocessing/wiki)", + "none": "If not applicable use (none)" + } +} diff --git a/src/egon/data/metadata/demand.egon_building_heat_peak_loads.json b/src/egon/data/metadata/demand.egon_building_heat_peak_loads.json new file mode 100644 index 000000000..17f7dfee0 --- /dev/null +++ b/src/egon/data/metadata/demand.egon_building_heat_peak_loads.json @@ -0,0 +1,209 @@ +{ + "name": "demand.egon_building_heat_peak_loads", + "title": "Heat peak loads for egon buildings", + "id": "WILL_BE_SET_AT_PUBLICATION", + "description": "Heat peak load table of residential and cts buildings for two scenarios ", + "language": "undefined", + "keywords": [], + "publicationDate": "2023-03-13", + "context": { + "homepage": "https://ego-n.org/", + "documentation": "https://egon-data.readthedocs.io/en/latest/", + "sourceCode": "https://github.com/openego/eGon-data", + "contact": "https://ego-n.org/partners/", + "grantNo": "03EI1002", + "fundingAgency": "Bundesministerium f\u00fcr Wirtschaft und Klimaschutz", + "fundingAgencyLogo": "https://commons.wikimedia.org/wiki/File:BMWi_Logo_2021.svg#/media/File:BMWi_Logo_2021.svg", + "publisherLogo": "https://ego-n.org/images/eGon_logo_noborder_transbg.svg" + }, + "spatial": { + "location": "", + "extent": "Germany", + "resolution": "Buildings" + }, + "temporal": { + "referenceDate": "2023-03-13", + "timeseries": {} + }, + "sources": [ + { + "title": "Data bundle for egon-data: A transparent and reproducible data processing pipeline for energy system modeling", + "description": "egon-data provides a transparent and reproducible open data based data processing pipeline for generating data models suitable for energy system modeling. The data is customized for the requirements of the research project eGon. The research project aims to develop tools for an open and cross-sectoral planning of transmission and distribution grids. For further information please visit the eGon project website or its Github repository. egon-data retrieves and processes data from several different external input sources. As not all data dependencies can be downloaded automatically from external sources we provide a data bundle to be downloaded by egon-data.", + "path": "https://doi.org/10.5281/zenodo.6630616", + "licenses": [ + { + "name": "CC-BY-4.0", + "title": "Creative Commons Attribution 4.0 International", + "path": "https://creativecommons.org/licenses/by/4.0/legalcode", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute.", + "attribution": "\u00a9 eGon-data developers, \u00a9 Leibniz Institute for Applied Geophysics, \u00a9 Fraunhofer IEE, \u00a9 BGR Hannover, \u00a9 Danielle Schmidt, \u00a9 \u00dcbertragungsnetzbetreiber, \u00a9 DIW Berlin, \u00a9 Reiner Lemoine Institut, \u00a9 Statistisches Bundesamt, Wiesbaden 2008, \u00a9 Statistische \u00c4mter des Bundes und der L\u00e4nder 2021" + } + ] + }, + { + "title": "eGon-data", + "description": "Workflow to download, process and generate data sets suitable for the further research conducted in the project eGon (https://ego-n.org/)", + "path": "https://github.com/openego/eGon-data", + "licenses": [ + { + "name": "AGPL-3.0 License", + "title": "GNU Affero General Public License v3.0", + "path": "https://www.gnu.org/licenses/agpl-3.0.de.html", + "instruction": "Permissions of this strongest copyleft license are conditioned on making available complete source code of licensed works and modifications, which include larger works using a licensed work, under the same license. Copyright and license notices must be preserved. Contributors provide an express grant of patent rights. When a modified version is used to provide a service over a network, the complete source code of the modified version must be made available.", + "attribution": "\u00a9 eGon-data developers" + } + ] + }, + { + "title": "OpenStreetMap Data Extracts (Geofabrik)", + "description": "Full data extract of OpenStreetMap data for defined spatial extent at 2022-01-01", + "path": "https://download.geofabrik.de/europe/germany-210101.osm.pbf", + "licenses": [ + { + "name": "ODbL-1.0", + "title": "Open Data Commons Open Database License 1.0", + "path": "https://opendatacommons.org/licenses/odbl/1.0/", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute, Share-Alike, Keep open!", + "attribution": "\u00a9 OpenStreetMap contributors" + } + ] + }, + { + "title": "Verwaltungsgebiete 1:250 000 (Ebenen)", + "description": "Der Datenbestand umfasst s\u00e4mtliche Verwaltungseinheiten der hierarchischen Verwaltungsebenen vom Staat bis zu den Gemeinden mit ihren Grenzen, statistischen Schl\u00fcsselzahlen, Namen der Verwaltungseinheit sowie die spezifische Bezeichnung der Verwaltungsebene des jeweiligen Landes.", + "path": "https://daten.gdz.bkg.bund.de/produkte/vg/vg250_ebenen_0101/2020/vg250_01-01.geo84.shape.ebenen.zip", + "licenses": [ + { + "name": "dl-by-de/2.0", + "title": "Datenlizenz Deutschland \u2013 Namensnennung \u2013 Version 2.0", + "path": "www.govdata.de/dl-de/by-2-0", + "instruction": "Jede Nutzung ist unter den Bedingungen dieser \u201eDatenlizenz Deutschland - Namensnennung - Version 2.0 zul\u00e4ssig.Die bereitgestellten Daten und Metadaten d\u00fcrfen f\u00fcr die kommerzielle und nicht kommerzielle Nutzung insbesondere:(1) vervielf\u00e4ltigt, ausgedruckt, pr\u00e4sentiert, ver\u00e4ndert, bearbeitet sowie an Dritte \u00fcbermittelt werden; (2) mit eigenen Daten und Daten Anderer zusammengef\u00fchrt und zu selbst\u00e4ndigen neuen Datens\u00e4tzen verbunden werden; (3) in interne und externe Gesch\u00e4ftsprozesse, Produkte und Anwendungen in \u00f6ffentlichen und nicht \u00f6ffentlichen elektronischen Netzwerken eingebunden werden.Bei der Nutzung ist sicherzustellen, dass folgende Angaben als Quellenvermerk enthalten sind:(1) Bezeichnung des Bereitstellers nach dessen Ma\u00dfgabe,(2) der Vermerk Datenlizenz Deutschland \u2013 Namensnennung \u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 sowie(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.Ver\u00e4nderungen, Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk mit dem Hinweis zu versehen, dass die Daten ge\u00e4ndert wurden.", + "attribution": "\u00a9 Bundesamt f\u00fcr Kartographie und Geod\u00e4sie 2020 (Daten ver\u00e4ndert)" + } + ] + }, + { + "title": "ERA5 global reanalysis", + "description": "ERA5 is the fifth generation ECMWF reanalysis for the global climate and weather for the past 4 to 7 decades. Currently data is available from 1950, split into Climate Data Store entries for 1950-1978 (preliminary back extension) and from 1979 onwards (final release plus timely updates, this page). ERA5 replaces the ERA-Interim reanalysis. See the online ERA5 documentation (https://confluence.ecmwf.int/display/CKB/ERA5%3A+data+documentation#ERA5:datadocumentation-Dataupdatefrequency) for more information.", + "path": "https://confluence.ecmwf.int/display/CKB/ERA5%3A+data+documentation#ERA5:datadocumentation-Dataupdatefrequency", + "licenses": [ + { + "name": "Licence to use Copernicus Products", + "title": "Licence to use Copernicus Products", + "path": "https://cds.climate.copernicus.eu/api/v2/terms/static/licence-to-use-copernicus-products.pdf", + "instruction": "This Licence is free of charge, worldwide, non-exclusive, royalty free and perpetual. Access to Copernicus Products is given for any purpose in so far as it is lawful, whereas use may include, but is not limited to: reproduction; distribution; communication to the public; adaptation, modification and combination with other data and information; or any combination of the foregoing", + "attribution": "\u00a9 Copernicus Climate Change Service (C3S) Climate Data Store" + } + ] + }, + { + "title": "Peta5 0 1 HD res", + "description": "modelled Heat Demand distribution (in GJ per hectare grid cell) for residential heat demands for space heating and hot water for the year 2015 using HRE4 data and the combined top-down bottom-up approach of HRE4", + "path": "https://s-eenergies-open-data-euf.hub.arcgis.com/search", + "licenses": [ + { + "name": "Creative Commons Attribution 4.0 International", + "title": "CC BY 4.0", + "path": "https://creativecommons.org/licenses/by/4.0/", + "instruction": "You are free: To Share, To Adapt; As long as you: Attribute!", + "attribution": "\\u00a9 Flensburg, Halmstad and Aalborg universities" + } + ] + }, + { + "title": "Peta5 0 1 HD ser", + "description": "modelled Heat Demand distribution (in GJ per hectare grid cell) for service-sector heat demands for space heating, hot water and process heat for the year 2015 using HRE4 data and the combined top-down bottom-up approach of HRE4", + "path": "https://s-eenergies-open-data-euf.hub.arcgis.com/search", + "licenses": [ + { + "name": "Creative Commons Attribution 4.0 International", + "title": "CC BY 4.0", + "path": "https://creativecommons.org/licenses/by/4.0/", + "instruction": "You are free: To Share, To Adapt; As long as you: Attribute!", + "attribution": "\\u00a9 Flensburg, Halmstad and Aalborg universities" + } + ] + } + ], + "licenses": [ + { + "name": "ODbL-1.0", + "title": "Open Data Commons Open Database License 1.0", + "path": "https://opendatacommons.org/licenses/odbl/1.0/", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute, Share-Alike, Keep open!", + "attribution": "\u00a9 Reiner Lemoine Institut" + } + ], + "contributors": [ + { + "title": "nailend", + "email": "https://github.com/nailend", + "date": "2023-03-13", + "object": "metadata", + "comment": "create metadata" + } + ], + "resources": [ + { + "profile": "tabular-data-resource", + "name": "demand.egon_building_heat_peak_loads", + "path": "", + "format": "PostgreSQL", + "encoding": "UTF-8", + "schema": { + "fields": [ + { + "name": "building_id", + "description": "Building id (reference to: openstreetmap.osm_buildings_filtered.id, openstreetmap.osm_buildings_synthetic.id, boundaries.egon_map_zensus_mvgd_building)", + "type": "integer", + "unit": "none" + }, + { + "name": "scenario", + "description": "Scenario (eGon100RE, eGon2035)", + "type": "character varying", + "unit": "none" + }, + { + "name": "sector", + "description": "Sector (cts, residential)", + "type": "characters varying", + "unit": "none" + }, + { + "name": "peak_load_in_w", + "description": "Peak load value", + "type": "real", + "unit": "W" + } + ], + "primaryKey": "building_id, scenario, sector" + }, + "dialect": { + "delimiter": "", + "decimalSeparator": "." + } + } + ], + "review": { + "path": "", + "badge": "" + }, + "metaMetadata": { + "metadataVersion": "OEP-1.4.1", + "metadataLicense": { + "name": "CC0-1.0", + "title": "Creative Commons Zero v1.0 Universal", + "path": "https://creativecommons.org/publicdomain/zero/1.0/" + } + }, + "_comment": { + "metadata": "Metadata documentation and explanation (https://github.com/OpenEnergyPlatform/oemetadata/blob/master/metadata/v141/metadata_key_description.md)", + "dates": "Dates and time must follow the ISO8601 including time zone (YYYY-MM-DD or YYYY-MM-DDThh:mm:ss\u00b1hh)", + "units": "Use a space between numbers and units (100 m)", + "languages": "Languages must follow the IETF (BCP47) format (en-GB, en-US, de-DE)", + "licenses": "License name must follow the SPDX License List (https://spdx.org/licenses/)", + "review": "Following the OEP Data Review (https://github.com/OpenEnergyPlatform/data-preprocessing/wiki)", + "none": "If not applicable use (none)" + } +} diff --git a/src/egon/data/metadata/demand.egon_cts_electricity_demand_building_share.json b/src/egon/data/metadata/demand.egon_cts_electricity_demand_building_share.json new file mode 100644 index 000000000..c7a60b34c --- /dev/null +++ b/src/egon/data/metadata/demand.egon_cts_electricity_demand_building_share.json @@ -0,0 +1,195 @@ +{ + "name": "demand.egon_cts_electricity_demand_building_share", + "title": "Electricity demand profile share of egon cts buildings", + "id": "WILL_BE_SET_AT_PUBLICATION", + "description": "Electricity demand profile share of egon cts buildings for two scenarios with bus_id", + "language": "undefined", + "keywords": [], + "publicationDate": "2023-03-13", + "context": { + "homepage": "https://ego-n.org/", + "documentation": "https://egon-data.readthedocs.io/en/latest/", + "sourceCode": "https://github.com/openego/eGon-data", + "contact": "https://ego-n.org/partners/", + "grantNo": "03EI1002", + "fundingAgency": "Bundesministerium f\u00fcr Wirtschaft und Klimaschutz", + "fundingAgencyLogo": "https://commons.wikimedia.org/wiki/File:BMWi_Logo_2021.svg#/media/File:BMWi_Logo_2021.svg", + "publisherLogo": "https://ego-n.org/images/eGon_logo_noborder_transbg.svg" + }, + "spatial": { + "location": "", + "extent": "Germany", + "resolution": "Buildings" + }, + "temporal": { + "referenceDate": "2023-03-13", + "timeseries": {} + }, + "sources": [ + { + "title": "Data bundle for egon-data: A transparent and reproducible data processing pipeline for energy system modeling", + "description": "egon-data provides a transparent and reproducible open data based data processing pipeline for generating data models suitable for energy system modeling. The data is customized for the requirements of the research project eGon. The research project aims to develop tools for an open and cross-sectoral planning of transmission and distribution grids. For further information please visit the eGon project website or its Github repository. egon-data retrieves and processes data from several different external input sources. As not all data dependencies can be downloaded automatically from external sources we provide a data bundle to be downloaded by egon-data.", + "path": "https://doi.org/10.5281/zenodo.6630616", + "licenses": [ + { + "name": "CC-BY-4.0", + "title": "Creative Commons Attribution 4.0 International", + "path": "https://creativecommons.org/licenses/by/4.0/legalcode", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute.", + "attribution": "\u00a9 eGon-data developers, \u00a9 Leibniz Institute for Applied Geophysics, \u00a9 Fraunhofer IEE, \u00a9 BGR Hannover, \u00a9 Danielle Schmidt, \u00a9 \u00dcbertragungsnetzbetreiber, \u00a9 DIW Berlin, \u00a9 Reiner Lemoine Institut, \u00a9 Statistisches Bundesamt, Wiesbaden 2008, \u00a9 Statistische \u00c4mter des Bundes und der L\u00e4nder 2021" + } + ] + }, + { + "title": "eGon-data", + "description": "Workflow to download, process and generate data sets suitable for the further research conducted in the project eGon (https://ego-n.org/)", + "path": "https://github.com/openego/eGon-data", + "licenses": [ + { + "name": "AGPL-3.0 License", + "title": "GNU Affero General Public License v3.0", + "path": "https://www.gnu.org/licenses/agpl-3.0.de.html", + "instruction": "Permissions of this strongest copyleft license are conditioned on making available complete source code of licensed works and modifications, which include larger works using a licensed work, under the same license. Copyright and license notices must be preserved. Contributors provide an express grant of patent rights. When a modified version is used to provide a service over a network, the complete source code of the modified version must be made available.", + "attribution": "\u00a9 eGon-data developers" + } + ] + }, + { + "title": "Statistisches Bundesamt (Destatis) - Ergebnisse des Zensus 2011 zum Download", + "description": "Als Download bieten wir Ihnen auf dieser Seite zus\\u00e4tzlich zur Zensusdatenbank CSV- und teilweise Excel-Tabellen mit umfassenden Personen-, Haushalts- und Familien- sowie Geb\\u00e4ude- und Wohnungsmerkmalen. Die Ergebnisse liegen auf Bundes-, L\\u00e4nder-, Kreis- und Gemeindeebene vor. Au\\u00dferdem sind einzelne Ergebnisse f\\u00fcr Gitterzellen verf\\u00fcgbar.", + "path": "https://www.zensus2011.de/DE/Home/Aktuelles/DemografischeGrunddaten.html", + "licenses": [ + { + "name": "dl-by-de/2.0", + "title": "Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0", + "path": "www.govdata.de/dl-de/by-2-0", + "instruction": "Jede Nutzung ist unter den Bedingungen dieser \\u201eDatenlizenz Deutschland - Namensnennung - Version 2.0 zul\\u00e4ssig.\\nDie bereitgestellten Daten und Metadaten d\\u00fcrfen f\\u00fcr die kommerzielle und nicht kommerzielle Nutzung insbesondere:(1) vervielf\\u00e4ltigt, ausgedruckt, pr\\u00e4sentiert, ver\\u00e4ndert, bearbeitet sowie an Dritte \\u00fcbermittelt werden;\\n (2) mit eigenen Daten und Daten Anderer zusammengef\\u00fchrt und zu selbst\\u00e4ndigen neuen Datens\\u00e4tzen verbunden werden;\\n (3) in interne und externe Gesch\\u00e4ftsprozesse, Produkte und Anwendungen in \\u00f6ffentlichen und nicht \\u00f6ffentlichen elektronischen Netzwerken eingebunden werden.\\nBei der Nutzung ist sicherzustellen, dass folgende Angaben als Quellenvermerk enthalten sind:\\n(1) Bezeichnung des Bereitstellers nach dessen Ma\\u00dfgabe,\\n(2) der Vermerk Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 sowie\\n(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.\\nVer\\u00e4nderungen, Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk mit dem Hinweis zu versehen, dass die Daten ge\\u00e4ndert wurden.", + "attribution": "\\u00a9 Statistische \\u00c4mter des Bundes und der L\\u00e4nder 2014" + } + ] + }, + { + "title": "Dokumentation - Zensus 2011 - Methoden und Verfahren", + "description": "Diese Publikation beschreibt ausf\\u00fchrlich die Methoden und Verfahren des registergest\\u00fctzten Zensus 2011; von der Datengewinnung und -aufbereitung bis hin zur Ergebniserstellung und Geheimhaltung. Der vorliegende Band wurde von den Statistischen \\u00c4mtern des Bundes und der L\\u00e4nder im Juni 2015 ver\\u00f6ffentlicht.", + "path": "https://www.destatis.de/DE/Publikationen/Thematisch/Bevoelkerung/Zensus/ZensusBuLaMethodenVerfahren5121105119004.pdf?__blob=publicationFile", + "licenses": [ + { + "name": "dl-by-de/2.0", + "title": "Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0", + "path": "www.govdata.de/dl-de/by-2-0", + "instruction": "Jede Nutzung ist unter den Bedingungen dieser \\u201eDatenlizenz Deutschland - Namensnennung - Version 2.0 zul\\u00e4ssig.\\nDie bereitgestellten Daten und Metadaten d\\u00fcrfen f\\u00fcr die kommerzielle und nicht kommerzielle Nutzung insbesondere:(1) vervielf\\u00e4ltigt, ausgedruckt, pr\\u00e4sentiert, ver\\u00e4ndert, bearbeitet sowie an Dritte \\u00fcbermittelt werden;\\n (2) mit eigenen Daten und Daten Anderer zusammengef\\u00fchrt und zu selbst\\u00e4ndigen neuen Datens\\u00e4tzen verbunden werden;\\n (3) in interne und externe Gesch\\u00e4ftsprozesse, Produkte und Anwendungen in \\u00f6ffentlichen und nicht \\u00f6ffentlichen elektronischen Netzwerken eingebunden werden.\\nBei der Nutzung ist sicherzustellen, dass folgende Angaben als Quellenvermerk enthalten sind:\\n(1) Bezeichnung des Bereitstellers nach dessen Ma\\u00dfgabe,\\n(2) der Vermerk Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 sowie\\n(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.\\nVer\\u00e4nderungen, Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk mit dem Hinweis zu versehen, dass die Daten ge\\u00e4ndert wurden.", + "attribution": "\\u00a9 Statistisches Bundesamt, Wiesbaden 2015 (im Auftrag der Herausgebergemeinschaft)" + } + ] + }, + { + "title": "OpenStreetMap Data Extracts (Geofabrik)", + "description": "Full data extract of OpenStreetMap data for defined spatial extent at 2022-01-01", + "path": "https://download.geofabrik.de/europe/germany-210101.osm.pbf", + "licenses": [ + { + "name": "ODbL-1.0", + "title": "Open Data Commons Open Database License 1.0", + "path": "https://opendatacommons.org/licenses/odbl/1.0/", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute, Share-Alike, Keep open!", + "attribution": "\u00a9 OpenStreetMap contributors" + } + ] + }, + { + "title": "DemandRegio", + "description": "Harmonisierung und Entwicklung von Verfahren zur regionalen und zeitlichen Aufl\u00f6sung von Energienachfragen", + "path": "https://doi.org/10.34805/ffe-119-20", + "license": [ + { + "name": "Creative Commons Attribution 4.0 International", + "title": "CC BY 4.0", + "path": "https://creativecommons.org/licenses/by/4.0/", + "instruction": "You are free: To Share, To Adapt; As long as you: Attribute!", + "attribution": "\u00a9 FZJ, TUB, FfE" + } + ] + } + ], + "licenses": [ + { + "name": "ODbL-1.0", + "title": "Open Data Commons Open Database License 1.0", + "path": "https://opendatacommons.org/licenses/odbl/1.0/", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute, Share-Alike, Keep open!", + "attribution": "\u00a9 Reiner Lemoine Institut" + } + ], + "contributors": [ + { + "title": "nailend", + "email": "https://github.com/nailend", + "date": "2023-03-13", + "object": "metadata", + "comment": "create metadata" + } + ], + "resources": [ + { + "profile": "tabular-data-resource", + "name": "demand.egon_cts_electricity_demand_building_share", + "path": "", + "format": "PostgreSQL", + "encoding": "UTF-8", + "schema": { + "fields": [ + { + "name": "building_id", + "description": "Building id (reference to: openstreetmap.osm_buildings_filtered.id, openstreetmap.osm_buildings_synthetic.id, boundaries.egon_map_zensus_mvgd_building)", + "type": "integer", + "unit": "none" + }, + { + "name": "scenario", + "description": "Scenario (eGon100RE, eGon2035) (reference: demand.egon_etrago_electricity_cts.scn_name)", + "type": "character varying", + "unit": "none" + }, + { + "name": "bus_id", + "description": "MVGD bus_id (reference to: demand.egon_etrago_electricity_cts.bus_id, grid.egon_hvmv_substation.bus_id, grid.egon_mv_grid_district.bus_id)", + "type": "integer", + "unit": "none" + }, + { + "name": "profile_share", + "description": "Profile share of MVGD elctricity profile", + "type": "real", + "unit": "None" + } + ], + "primaryKey": "building_id, scenario" + }, + "dialect": { + "delimiter": "", + "decimalSeparator": "." + } + } + ], + "review": { + "path": "", + "badge": "" + }, + "metaMetadata": { + "metadataVersion": "OEP-1.4.1", + "metadataLicense": { + "name": "CC0-1.0", + "title": "Creative Commons Zero v1.0 Universal", + "path": "https://creativecommons.org/publicdomain/zero/1.0/" + } + }, + "_comment": { + "metadata": "Metadata documentation and explanation (https://github.com/OpenEnergyPlatform/oemetadata/blob/master/metadata/v141/metadata_key_description.md)", + "dates": "Dates and time must follow the ISO8601 including time zone (YYYY-MM-DD or YYYY-MM-DDThh:mm:ss\u00b1hh)", + "units": "Use a space between numbers and units (100 m)", + "languages": "Languages must follow the IETF (BCP47) format (en-GB, en-US, de-DE)", + "licenses": "License name must follow the SPDX License List (https://spdx.org/licenses/)", + "review": "Following the OEP Data Review (https://github.com/OpenEnergyPlatform/data-preprocessing/wiki)", + "none": "If not applicable use (none)" + } +} diff --git a/src/egon/data/metadata/demand.egon_cts_heat_demand_building_share.json b/src/egon/data/metadata/demand.egon_cts_heat_demand_building_share.json new file mode 100644 index 000000000..0f9228270 --- /dev/null +++ b/src/egon/data/metadata/demand.egon_cts_heat_demand_building_share.json @@ -0,0 +1,181 @@ +{ + "name": "demand.egon_cts_heat_demand_building_share", + "title": "Heat demand profile share of egon cts buildings", + "id": "WILL_BE_SET_AT_PUBLICATION", + "description": "Heat demand profile share of egon cts buildings for two scenarios with bus_id", + "language": "undefined", + "keywords": [], + "publicationDate": "2023-03-13", + "context": { + "homepage": "https://ego-n.org/", + "documentation": "https://egon-data.readthedocs.io/en/latest/", + "sourceCode": "https://github.com/openego/eGon-data", + "contact": "https://ego-n.org/partners/", + "grantNo": "03EI1002", + "fundingAgency": "Bundesministerium f\u00fcr Wirtschaft und Klimaschutz", + "fundingAgencyLogo": "https://commons.wikimedia.org/wiki/File:BMWi_Logo_2021.svg#/media/File:BMWi_Logo_2021.svg", + "publisherLogo": "https://ego-n.org/images/eGon_logo_noborder_transbg.svg" + }, + "spatial": { + "location": "", + "extent": "Germany", + "resolution": "Buildings" + }, + "temporal": { + "referenceDate": "2023-03-13", + "timeseries": {} + }, + "sources": [ + { + "title": "Data bundle for egon-data: A transparent and reproducible data processing pipeline for energy system modeling", + "description": "egon-data provides a transparent and reproducible open data based data processing pipeline for generating data models suitable for energy system modeling. The data is customized for the requirements of the research project eGon. The research project aims to develop tools for an open and cross-sectoral planning of transmission and distribution grids. For further information please visit the eGon project website or its Github repository. egon-data retrieves and processes data from several different external input sources. As not all data dependencies can be downloaded automatically from external sources we provide a data bundle to be downloaded by egon-data.", + "path": "https://doi.org/10.5281/zenodo.6630616", + "licenses": [ + { + "name": "CC-BY-4.0", + "title": "Creative Commons Attribution 4.0 International", + "path": "https://creativecommons.org/licenses/by/4.0/legalcode", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute.", + "attribution": "\u00a9 eGon-data developers, \u00a9 Leibniz Institute for Applied Geophysics, \u00a9 Fraunhofer IEE, \u00a9 BGR Hannover, \u00a9 Danielle Schmidt, \u00a9 \u00dcbertragungsnetzbetreiber, \u00a9 DIW Berlin, \u00a9 Reiner Lemoine Institut, \u00a9 Statistisches Bundesamt, Wiesbaden 2008, \u00a9 Statistische \u00c4mter des Bundes und der L\u00e4nder 2021" + } + ] + }, + { + "title": "eGon-data", + "description": "Workflow to download, process and generate data sets suitable for the further research conducted in the project eGon (https://ego-n.org/)", + "path": "https://github.com/openego/eGon-data", + "licenses": [ + { + "name": "AGPL-3.0 License", + "title": "GNU Affero General Public License v3.0", + "path": "https://www.gnu.org/licenses/agpl-3.0.de.html", + "instruction": "Permissions of this strongest copyleft license are conditioned on making available complete source code of licensed works and modifications, which include larger works using a licensed work, under the same license. Copyright and license notices must be preserved. Contributors provide an express grant of patent rights. When a modified version is used to provide a service over a network, the complete source code of the modified version must be made available.", + "attribution": "\u00a9 eGon-data developers" + } + ] + }, + { + "title": "OpenStreetMap Data Extracts (Geofabrik)", + "description": "Full data extract of OpenStreetMap data for defined spatial extent at 2022-01-01", + "path": "https://download.geofabrik.de/europe/germany-210101.osm.pbf", + "licenses": [ + { + "name": "ODbL-1.0", + "title": "Open Data Commons Open Database License 1.0", + "path": "https://opendatacommons.org/licenses/odbl/1.0/", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute, Share-Alike, Keep open!", + "attribution": "\u00a9 OpenStreetMap contributors" + } + ] + }, + { + "title": "Peta5 0 1 HD ser", + "description": "modelled Heat Demand distribution (in GJ per hectare grid cell) for service-sector heat demands for space heating, hot water and process heat for the year 2015 using HRE4 data and the combined top-down bottom-up approach of HRE4", + "path": "https://s-eenergies-open-data-euf.hub.arcgis.com/search", + "licenses": [ + { + "name": "Creative Commons Attribution 4.0 International", + "title": "CC BY 4.0", + "path": "https://creativecommons.org/licenses/by/4.0/", + "instruction": "You are free: To Share, To Adapt; As long as you: Attribute!", + "attribution": "\\u00a9 Flensburg, Halmstad and Aalborg universities" + } + ] + }, + { + "title": "Peta5 0 1 HD res", + "description": "modelled Heat Demand distribution (in GJ per hectare grid cell) for residential heat demands for space heating and hot water for the year 2015 using HRE4 data and the combined top-down bottom-up approach of HRE4", + "path": "https://s-eenergies-open-data-euf.hub.arcgis.com/search", + "licenses": [ + { + "name": "Creative Commons Attribution 4.0 International", + "title": "CC BY 4.0", + "path": "https://creativecommons.org/licenses/by/4.0/", + "instruction": "You are free: To Share, To Adapt; As long as you: Attribute!", + "attribution": "\\u00a9 Flensburg, Halmstad and Aalborg universities" + } + ] + } + ], + "licenses": [ + { + "name": "ODbL-1.0", + "title": "Open Data Commons Open Database License 1.0", + "path": "https://opendatacommons.org/licenses/odbl/1.0/", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute, Share-Alike, Keep open!", + "attribution": "\u00a9 Reiner Lemoine Institut" + } + ], + "contributors": [ + { + "title": "nailend", + "email": "https://github.com/nailend", + "date": "2023-03-13", + "object": "metadata", + "comment": "create metadata" + } + ], + "resources": [ + { + "profile": "tabular-data-resource", + "name": "demand.egon_cts_heat_demand_building_share", + "path": "", + "format": "PostgreSQL", + "encoding": "UTF-8", + "schema": { + "fields": [ + { + "name": "building_id", + "description": "Building id (reference to: openstreetmap.osm_buildings_filtered.id, openstreetmap.osm_buildings_synthetic.id, boundaries.egon_map_zensus_mvgd_building)", + "type": "integer", + "unit": "none" + }, + { + "name": "scenario", + "description": "Scenario (eGon100RE, eGon2035)", + "type": "character varying", + "unit": "none" + }, + { + "name": "bus_id", + "description": "MVGD bus_id (reference to: demand.egon_etrago_heat_cts.bus_id, grid.egon_hvmv_substation.bus_id, grid.egon_mv_grid_district.bus_id)", + "type": "integer", + "unit": "none" + }, + { + "name": "profile_share", + "description": "Profile share of MVGD heat profile ", + "type": "real", + "unit": "None" + } + ], + "primaryKey": "building_id, scenario" + }, + "dialect": { + "delimiter": "", + "decimalSeparator": "." + } + } + ], + "review": { + "path": "", + "badge": "" + }, + "metaMetadata": { + "metadataVersion": "OEP-1.4.1", + "metadataLicense": { + "name": "CC0-1.0", + "title": "Creative Commons Zero v1.0 Universal", + "path": "https://creativecommons.org/publicdomain/zero/1.0/" + } + }, + "_comment": { + "metadata": "Metadata documentation and explanation (https://github.com/OpenEnergyPlatform/oemetadata/blob/master/metadata/v141/metadata_key_description.md)", + "dates": "Dates and time must follow the ISO8601 including time zone (YYYY-MM-DD or YYYY-MM-DDThh:mm:ss\u00b1hh)", + "units": "Use a space between numbers and units (100 m)", + "languages": "Languages must follow the IETF (BCP47) format (en-GB, en-US, de-DE)", + "licenses": "License name must follow the SPDX License List (https://spdx.org/licenses/)", + "review": "Following the OEP Data Review (https://github.com/OpenEnergyPlatform/data-preprocessing/wiki)", + "none": "If not applicable use (none)" + } +} diff --git a/src/egon/data/metadata/demand.egon_daily_heat_demand_per_climate_zone.json b/src/egon/data/metadata/demand.egon_daily_heat_demand_per_climate_zone.json new file mode 100644 index 000000000..06d950eda --- /dev/null +++ b/src/egon/data/metadata/demand.egon_daily_heat_demand_per_climate_zone.json @@ -0,0 +1,163 @@ +{ + "name": "demand.egon_daily_heat_demand_per_climate_zone", + "title": "eGon daily heat demand share per climate zone", + "id": "WILL_BE_SET_AT_PUBLICATION", + "description": "Share of daily heat demands per climate zone", + "language": [ + "en-EN", + "de-DE" + ], + "keywords": [], + "publicationDate": "2022-09-14", + "context": { + "homepage": "https://ego-n.org/", + "documentation": "https://egon-data.readthedocs.io/en/latest/", + "sourceCode": "https://github.com/openego/eGon-data", + "contact": "https://ego-n.org/partners/", + "grantNo": "03EI1002", + "fundingAgency": "Bundesministerium f\u00fcr Wirtschaft und Klimaschutz", + "fundingAgencyLogo": "https://www.innovation-beratung-foerderung.de/INNO/Redaktion/DE/Bilder/Titelbilder/titel_foerderlogo_bmwi.jpg?__blob=normal&v=3", + "publisherLogo": "https://ego-n.org/images/eGon_logo_noborder_transbg.svg" + }, + "spatial": { + "location": "", + "extent": "Germany", + "resolution": "climate zones" + }, + "temporal": { + "referenceDate": "2022-09-14", + "timeseries": {} + }, + "sources": [ + { + "title": "eGon-data", + "description": "Workflow to download, process and generate data sets suitable for the further research conducted in the project eGon (https://ego-n.org/)", + "path": "https://github.com/openego/eGon-data", + "licenses": [ + { + "name": "AGPL-3.0 License", + "title": "GNU Affero General Public License v3.0", + "path": "https://www.gnu.org/licenses/agpl-3.0.de.html", + "instruction": "Permissions of this strongest copyleft license are conditioned on making available complete source code of licensed works and modifications, which include larger works using a licensed work, under the same license. Copyright and license notices must be preserved. Contributors provide an express grant of patent rights. When a modified version is used to provide a service over a network, the complete source code of the modified version must be made available.", + "attribution": "\u00a9 eGon-data developers" + } + ] + }, + { + "title": "Data bundle for egon-data: A transparent and reproducible data processing pipeline for energy system modeling", + "description": "egon-data provides a transparent and reproducible open data based data processing pipeline for generating data models suitable for energy system modeling. The data is customized for the requirements of the research project eGon. The research project aims to develop tools for an open and cross-sectoral planning of transmission and distribution grids. For further information please visit the eGon project website or its Github repository. egon-data retrieves and processes data from several different external input sources. As not all data dependencies can be downloaded automatically from external sources we provide a data bundle to be downloaded by egon-data.", + "path": "https://doi.org/10.5281/zenodo.6630616", + "licenses": [ + { + "name": "CC-BY-4.0", + "title": "Creative Commons Attribution 4.0 International", + "path": "https://creativecommons.org/licenses/by/4.0/legalcode", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute.", + "attribution": "\u00a9 eGon-data developers, \u00a9 Leibniz Institute for Applied Geophysics, \u00a9 Fraunhofer IEE, \u00a9 BGR Hannover, \u00a9 Danielle Schmidt, \u00a9 \u00dcbertragungsnetzbetreiber, \u00a9 DIW Berlin, \u00a9 Reiner Lemoine Institut, \u00a9 Statistisches Bundesamt, Wiesbaden 2008, \u00a9 Statistische \u00c4mter des Bundes und der L\u00e4nder 2021" + } + ] + }, + { + "title": "ERA5 global reanalysis", + "description": "ERA5 is the fifth generation ECMWF reanalysis for the global climate and weather for the past 4 to 7 decades. Currently data is available from 1950, split into Climate Data Store entries for 1950-1978 (preliminary back extension) and from 1979 onwards (final release plus timely updates, this page). ERA5 replaces the ERA-Interim reanalysis. See the online ERA5 documentation (https://confluence.ecmwf.int/display/CKB/ERA5%3A+data+documentation#ERA5:datadocumentation-Dataupdatefrequency) for more information.", + "path": "https://confluence.ecmwf.int/display/CKB/ERA5%3A+data+documentation#ERA5:datadocumentation-Dataupdatefrequency", + "licenses": [ + { + "name": "Licence to use Copernicus Products", + "title": "Licence to use Copernicus Products", + "path": "https://cds.climate.copernicus.eu/api/v2/terms/static/licence-to-use-copernicus-products.pdf", + "instruction": "This Licence is free of charge, worldwide, non-exclusive, royalty free and perpetual. Access to Copernicus Products is given for any purpose in so far as it is lawful, whereas use may include, but is not limited to: reproduction; distribution; communication to the public; adaptation, modification and combination with other data and information; or any combination of the foregoing", + "attribution": "\u00a9 Copernicus Climate Change Service (C3S) Climate Data Store" + } + ] + } + ], + "licenses": [ + { + "name": "CC-BY-4.0", + "title": "Creative Commons Attribution 4.0 International", + "path": "https://creativecommons.org/licenses/by/4.0/legalcode", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute.", + "attribution": "\u00a9 eGon-data developers" + } + ], + "contributors": [ + { + "title": "ClaraBuettner", + "email": "https://github.com/ClaraBuettner", + "date": "2022-09-12", + "object": "metadata", + "comment": "Create metadata" + }, + { + "title": "nesnoj", + "email": "https://github.com/nesnoj", + "date": "2022-09-14", + "object": "metadata", + "comment": "Update metadata" + } + ], + "resources": [ + { + "profile": "tabular-data-resource", + "name": "demand.egon_daily_heat_demand_per_climate_zone", + "path": "", + "format": "PostgreSQL", + "encoding": "UTF-8", + "schema": { + "fields": [ + { + "name": "climate_zone", + "description": "Name of the corresponding climate zone", + "type": "str", + "unit": "none" + }, + { + "name": "daily_demand_share", + "description": "Share of annual heat demand for each day", + "type": "double precision", + "unit": "per unit" + }, + { + "name": "day_of_year", + "description": "Number of day in the year", + "type": "integer", + "unit": "none" + }, + { + "name": "temperature_class", + "description": "Temperature class (considering the mean outside temeperature during the day)", + "type": "integer", + "unit": "none" + } + ], + "primaryKey": "climate_zone" + }, + "dialect": { + "delimiter": "", + "decimalSeparator": "" + } + } + ], + "review": { + "path": "", + "badge": "" + }, + "metaMetadata": { + "metadataVersion": "OEP-1.4.1", + "metadataLicense": { + "name": "CC0-1.0", + "title": "Creative Commons Zero v1.0 Universal", + "path": "https://creativecommons.org/publicdomain/zero/1.0/" + } + }, + "_comment": { + "metadata": "Metadata documentation and explanation (https://github.com/OpenEnergyPlatform/oemetadata/blob/master/metadata/v141/metadata_key_description.md)", + "dates": "Dates and time must follow the ISO8601 including time zone (YYYY-MM-DD or YYYY-MM-DDThh:mm:ss\u00b1hh)", + "units": "Use a space between numbers and units (100 m)", + "languages": "Languages must follow the IETF (BCP47) format (en-GB, en-US, de-DE)", + "licenses": "License name must follow the SPDX License List (https://spdx.org/licenses/)", + "review": "Following the OEP Data Review (https://github.com/OpenEnergyPlatform/data-preprocessing/wiki)", + "none": "If not applicable use (none)" + } +} diff --git a/src/egon/data/metadata/demand.egon_etrago_electricity_cts.json b/src/egon/data/metadata/demand.egon_etrago_electricity_cts.json new file mode 100644 index 000000000..5db7cd176 --- /dev/null +++ b/src/egon/data/metadata/demand.egon_etrago_electricity_cts.json @@ -0,0 +1,197 @@ +{ + "name": "demand.egon_etrago_electricity_cts", + "title": "MV cts electricity demand profiles", + "id": "WILL_BE_SET_AT_PUBLICATION", + "description": "Aggregated electricity demand profile of cts sector at mvgd bus level for two scenarios. Values are given in MWh.", + "language": "undefined", + "keywords": [ + "" + ], + "publicationDate": "2022-09-14", + "context": { + "homepage": "https://ego-n.org/", + "documentation": "https://egon-data.readthedocs.io/en/latest/", + "sourceCode": "https://github.com/openego/eGon-data", + "contact": "https://ego-n.org/partners/", + "grantNo": "03EI1002", + "fundingAgency": "Bundesministerium f\u00fcr Wirtschaft und Klimaschutz", + "fundingAgencyLogo": "https://commons.wikimedia.org/wiki/File:BMWi_Logo_2021.svg#/media/File:BMWi_Logo_2021.svg", + "publisherLogo": "https://ego-n.org/images/eGon_logo_noborder_transbg.svg" + }, + "spatial": { + "location": "", + "extent": "Germany", + "resolution": "MV Grid" + }, + "temporal": { + "referenceDate": "2023-03-13", + "timeseries": { + "start": "2016-01-01", + "end": "2016-12-31", + "resolution": "1 h", + "alignment": "left", + "aggregationType": "sum" + } + }, + "sources": [ + { + "title": "Data bundle for egon-data: A transparent and reproducible data processing pipeline for energy system modeling", + "description": "egon-data provides a transparent and reproducible open data based data processing pipeline for generating data models suitable for energy system modeling. The data is customized for the requirements of the research project eGon. The research project aims to develop tools for an open and cross-sectoral planning of transmission and distribution grids. For further information please visit the eGon project website or its Github repository. egon-data retrieves and processes data from several different external input sources. As not all data dependencies can be downloaded automatically from external sources we provide a data bundle to be downloaded by egon-data.", + "path": "https://doi.org/10.5281/zenodo.6630616", + "licenses": [ + { + "name": "CC-BY-4.0", + "title": "Creative Commons Attribution 4.0 International", + "path": "https://creativecommons.org/licenses/by/4.0/legalcode", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute.", + "attribution": "\u00a9 eGon-data developers, \u00a9 Leibniz Institute for Applied Geophysics, \u00a9 Fraunhofer IEE, \u00a9 BGR Hannover, \u00a9 Danielle Schmidt, \u00a9 \u00dcbertragungsnetzbetreiber, \u00a9 DIW Berlin, \u00a9 Reiner Lemoine Institut, \u00a9 Statistisches Bundesamt, Wiesbaden 2008, \u00a9 Statistische \u00c4mter des Bundes und der L\u00e4nder 2021" + } + ] + }, + { + "title": "eGon-data", + "description": "Workflow to download, process and generate data sets suitable for the further research conducted in the project eGon (https://ego-n.org/)", + "path": "https://github.com/openego/eGon-data", + "licenses": [ + { + "name": "AGPL-3.0 License", + "title": "GNU Affero General Public License v3.0", + "path": "https://www.gnu.org/licenses/agpl-3.0.de.html", + "instruction": "Permissions of this strongest copyleft license are conditioned on making available complete source code of licensed works and modifications, which include larger works using a licensed work, under the same license. Copyright and license notices must be preserved. Contributors provide an express grant of patent rights. When a modified version is used to provide a service over a network, the complete source code of the modified version must be made available.", + "attribution": "\u00a9 eGon-data developers" + } + ] + }, + { + "title": "OpenStreetMap Data Extracts (Geofabrik)", + "description": "Full data extract of OpenStreetMap data for defined spatial extent at 2022-01-01", + "path": "https://download.geofabrik.de/europe/germany-210101.osm.pbf", + "licenses": [ + { + "name": "ODbL-1.0", + "title": "Open Data Commons Open Database License 1.0", + "path": "https://opendatacommons.org/licenses/odbl/1.0/", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute, Share-Alike, Keep open!", + "attribution": "\u00a9 OpenStreetMap contributors" + } + ] + }, + { + "title": "Statistisches Bundesamt (Destatis) - Ergebnisse des Zensus 2011 zum Download", + "description": "Als Download bieten wir Ihnen auf dieser Seite zus\\u00e4tzlich zur Zensusdatenbank CSV- und teilweise Excel-Tabellen mit umfassenden Personen-, Haushalts- und Familien- sowie Geb\\u00e4ude- und Wohnungsmerkmalen. Die Ergebnisse liegen auf Bundes-, L\\u00e4nder-, Kreis- und Gemeindeebene vor. Au\\u00dferdem sind einzelne Ergebnisse f\\u00fcr Gitterzellen verf\\u00fcgbar.", + "path": "https://www.zensus2011.de/DE/Home/Aktuelles/DemografischeGrunddaten.html", + "licenses": [ + { + "name": "dl-by-de/2.0", + "title": "Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0", + "path": "www.govdata.de/dl-de/by-2-0", + "instruction": "Jede Nutzung ist unter den Bedingungen dieser \\u201eDatenlizenz Deutschland - Namensnennung - Version 2.0 zul\\u00e4ssig.\\nDie bereitgestellten Daten und Metadaten d\\u00fcrfen f\\u00fcr die kommerzielle und nicht kommerzielle Nutzung insbesondere:(1) vervielf\\u00e4ltigt, ausgedruckt, pr\\u00e4sentiert, ver\\u00e4ndert, bearbeitet sowie an Dritte \\u00fcbermittelt werden;\\n (2) mit eigenen Daten und Daten Anderer zusammengef\\u00fchrt und zu selbst\\u00e4ndigen neuen Datens\\u00e4tzen verbunden werden;\\n (3) in interne und externe Gesch\\u00e4ftsprozesse, Produkte und Anwendungen in \\u00f6ffentlichen und nicht \\u00f6ffentlichen elektronischen Netzwerken eingebunden werden.\\nBei der Nutzung ist sicherzustellen, dass folgende Angaben als Quellenvermerk enthalten sind:\\n(1) Bezeichnung des Bereitstellers nach dessen Ma\\u00dfgabe,\\n(2) der Vermerk Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 sowie\\n(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.\\nVer\\u00e4nderungen, Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk mit dem Hinweis zu versehen, dass die Daten ge\\u00e4ndert wurden.", + "attribution": "\\u00a9 Statistische \\u00c4mter des Bundes und der L\\u00e4nder 2014" + } + ] + }, + { + "title": "Dokumentation - Zensus 2011 - Methoden und Verfahren", + "description": "Diese Publikation beschreibt ausf\\u00fchrlich die Methoden und Verfahren des registergest\\u00fctzten Zensus 2011; von der Datengewinnung und -aufbereitung bis hin zur Ergebniserstellung und Geheimhaltung. Der vorliegende Band wurde von den Statistischen \\u00c4mtern des Bundes und der L\\u00e4nder im Juni 2015 ver\\u00f6ffentlicht.", + "path": "https://www.destatis.de/DE/Publikationen/Thematisch/Bevoelkerung/Zensus/ZensusBuLaMethodenVerfahren5121105119004.pdf?__blob=publicationFile", + "licenses": [ + { + "name": "dl-by-de/2.0", + "title": "Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0", + "path": "www.govdata.de/dl-de/by-2-0", + "instruction": "Jede Nutzung ist unter den Bedingungen dieser \\u201eDatenlizenz Deutschland - Namensnennung - Version 2.0 zul\\u00e4ssig.\\nDie bereitgestellten Daten und Metadaten d\\u00fcrfen f\\u00fcr die kommerzielle und nicht kommerzielle Nutzung insbesondere:(1) vervielf\\u00e4ltigt, ausgedruckt, pr\\u00e4sentiert, ver\\u00e4ndert, bearbeitet sowie an Dritte \\u00fcbermittelt werden;\\n (2) mit eigenen Daten und Daten Anderer zusammengef\\u00fchrt und zu selbst\\u00e4ndigen neuen Datens\\u00e4tzen verbunden werden;\\n (3) in interne und externe Gesch\\u00e4ftsprozesse, Produkte und Anwendungen in \\u00f6ffentlichen und nicht \\u00f6ffentlichen elektronischen Netzwerken eingebunden werden.\\nBei der Nutzung ist sicherzustellen, dass folgende Angaben als Quellenvermerk enthalten sind:\\n(1) Bezeichnung des Bereitstellers nach dessen Ma\\u00dfgabe,\\n(2) der Vermerk Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 sowie\\n(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.\\nVer\\u00e4nderungen, Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk mit dem Hinweis zu versehen, dass die Daten ge\\u00e4ndert wurden.", + "attribution": "\\u00a9 Statistisches Bundesamt, Wiesbaden 2015 (im Auftrag der Herausgebergemeinschaft)" + } + ] + }, + { + "title": "DemandRegio", + "description": "Harmonisierung und Entwicklung von Verfahren zur regionalen und zeitlichen Aufl\u00f6sung von Energienachfragen", + "path": "https://doi.org/10.34805/ffe-119-20", + "license": [ + { + "name": "Creative Commons Attribution 4.0 International", + "title": "CC BY 4.0", + "path": "https://creativecommons.org/licenses/by/4.0/", + "instruction": "You are free: To Share, To Adapt; As long as you: Attribute!", + "attribution": "\u00a9 FZJ, TUB, FfE" + } + ] + } + ], + "licenses": [ + { + "name": "ODbL-1.0", + "title": "Open Data Commons Open Database License 1.0", + "path": "https://opendatacommons.org/licenses/odbl/1.0/", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute, Share-Alike, Keep open!", + "attribution": "\u00a9 Reiner Lemoine Institut" + } + ], + "contributors": [ + { + "title": "nailend", + "email": "https://github.com/nailend", + "date": "2023-03-13", + "object": "metadata", + "comment": "Create metadata" + } + ], + "resources": [ + { + "profile": "tabular-data-resource", + "name": "demand.egon_etrago_electricity_cts", + "path": "", + "format": "PostgreSQL", + "encoding": "UTF-8", + "schema": { + "fields": [ + { + "name": "bus_id", + "description": "MV bus_id (reference: demand.egon_cts_electricity_demand_building_share.bus_id, grid.egon_hvmv_substation.bus_id, grid.egon_mv_grid_district.bus_id))", + "type": "integer", + "unit": "none" + }, + { + "name": "scn_name", + "description": "Scenario name (reference: demand.egon_cts_electricity_demand_building_share.scenario)", + "type": "character varying", + "unit": "none" + }, + { + "name": "p_set", + "description": "Cts electricity demand load timeseries for one year (8760 hours) in MWh", + "type": "array of real", + "unit": "MWh" + } + ], + "primaryKey": "id, scn_name" + }, + "dialect": { + "delimiter": "", + "decimalSeparator": "." + } + } + ], + "review": { + "path": "", + "badge": "" + }, + "metaMetadata": { + "metadataVersion": "OEP-1.4.1", + "metadataLicense": { + "name": "CC0-1.0", + "title": "Creative Commons Zero v1.0 Universal", + "path": "https://creativecommons.org/publicdomain/zero/1.0/" + } + }, + "_comment": { + "metadata": "Metadata documentation and explanation (https://github.com/OpenEnergyPlatform/oemetadata/blob/master/metadata/v141/metadata_key_description.md)", + "dates": "Dates and time must follow the ISO8601 including time zone (YYYY-MM-DD or YYYY-MM-DDThh:mm:ss\u00b1hh)", + "units": "Use a space between numbers and units (100 m)", + "languages": "Languages must follow the IETF (BCP47) format (en-GB, en-US, de-DE)", + "licenses": "License name must follow the SPDX License List (https://spdx.org/licenses/)", + "review": "Following the OEP Data Review (https://github.com/OpenEnergyPlatform/data-preprocessing/wiki)", + "none": "If not applicable use (none)" + } +} diff --git a/src/egon/data/metadata/demand.egon_etrago_electricity_households.json b/src/egon/data/metadata/demand.egon_etrago_electricity_households.json new file mode 100644 index 000000000..b46c4b4ef --- /dev/null +++ b/src/egon/data/metadata/demand.egon_etrago_electricity_households.json @@ -0,0 +1,197 @@ +{ + "name": "demand.egon_etrago_electricity_households", + "title": "MV residential electricity demand profiles", + "id": "WILL_BE_SET_AT_PUBLICATION", + "description": "Aggregated electricity demand profile of residential sector at mvgd bus level for two scenarios. Values are given in MWh.", + "language": "en-US", + "keywords": [ + "" + ], + "publicationDate": "2022-09-14", + "context": { + "homepage": "https://ego-n.org/", + "documentation": "https://egon-data.readthedocs.io/en/latest/", + "sourceCode": "https://github.com/openego/eGon-data", + "contact": "https://ego-n.org/partners/", + "grantNo": "03EI1002", + "fundingAgency": "Bundesministerium f\u00fcr Wirtschaft und Klimaschutz", + "fundingAgencyLogo": "https://commons.wikimedia.org/wiki/File:BMWi_Logo_2021.svg#/media/File:BMWi_Logo_2021.svg", + "publisherLogo": "https://ego-n.org/images/eGon_logo_noborder_transbg.svg" + }, + "spatial": { + "location": "", + "extent": "Germany", + "resolution": "MV Grid" + }, + "temporal": { + "referenceDate": "2023-03-13", + "timeseries": { + "start": "2016-01-01", + "end": "2016-12-31", + "resolution": "1 h", + "alignment": "left", + "aggregationType": "sum" + } + }, + "sources": [ + { + "title": "Data bundle for egon-data: A transparent and reproducible data processing pipeline for energy system modeling", + "description": "egon-data provides a transparent and reproducible open data based data processing pipeline for generating data models suitable for energy system modeling. The data is customized for the requirements of the research project eGon. The research project aims to develop tools for an open and cross-sectoral planning of transmission and distribution grids. For further information please visit the eGon project website or its Github repository. egon-data retrieves and processes data from several different external input sources. As not all data dependencies can be downloaded automatically from external sources we provide a data bundle to be downloaded by egon-data.", + "path": "https://doi.org/10.5281/zenodo.6630616", + "licenses": [ + { + "name": "CC-BY-4.0", + "title": "Creative Commons Attribution 4.0 International", + "path": "https://creativecommons.org/licenses/by/4.0/legalcode", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute.", + "attribution": "\u00a9 eGon-data developers, \u00a9 Leibniz Institute for Applied Geophysics, \u00a9 Fraunhofer IEE, \u00a9 BGR Hannover, \u00a9 Danielle Schmidt, \u00a9 \u00dcbertragungsnetzbetreiber, \u00a9 DIW Berlin, \u00a9 Reiner Lemoine Institut, \u00a9 Statistisches Bundesamt, Wiesbaden 2008, \u00a9 Statistische \u00c4mter des Bundes und der L\u00e4nder 2021" + } + ] + }, + { + "title": "eGon-data", + "description": "Workflow to download, process and generate data sets suitable for the further research conducted in the project eGon (https://ego-n.org/)", + "path": "https://github.com/openego/eGon-data", + "licenses": [ + { + "name": "AGPL-3.0 License", + "title": "GNU Affero General Public License v3.0", + "path": "https://www.gnu.org/licenses/agpl-3.0.de.html", + "instruction": "Permissions of this strongest copyleft license are conditioned on making available complete source code of licensed works and modifications, which include larger works using a licensed work, under the same license. Copyright and license notices must be preserved. Contributors provide an express grant of patent rights. When a modified version is used to provide a service over a network, the complete source code of the modified version must be made available.", + "attribution": "\u00a9 eGon-data developers" + } + ] + }, + { + "title": "OpenStreetMap Data Extracts (Geofabrik)", + "description": "Full data extract of OpenStreetMap data for defined spatial extent at 2022-01-01", + "path": "https://download.geofabrik.de/europe/germany-210101.osm.pbf", + "licenses": [ + { + "name": "ODbL-1.0", + "title": "Open Data Commons Open Database License 1.0", + "path": "https://opendatacommons.org/licenses/odbl/1.0/", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute, Share-Alike, Keep open!", + "attribution": "\u00a9 OpenStreetMap contributors" + } + ] + }, + { + "title": "Statistisches Bundesamt (Destatis) - Ergebnisse des Zensus 2011 zum Download", + "description": "Als Download bieten wir Ihnen auf dieser Seite zus\\u00e4tzlich zur Zensusdatenbank CSV- und teilweise Excel-Tabellen mit umfassenden Personen-, Haushalts- und Familien- sowie Geb\\u00e4ude- und Wohnungsmerkmalen. Die Ergebnisse liegen auf Bundes-, L\\u00e4nder-, Kreis- und Gemeindeebene vor. Au\\u00dferdem sind einzelne Ergebnisse f\\u00fcr Gitterzellen verf\\u00fcgbar.", + "path": "https://www.zensus2011.de/DE/Home/Aktuelles/DemografischeGrunddaten.html", + "licenses": [ + { + "name": "dl-by-de/2.0", + "title": "Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0", + "path": "www.govdata.de/dl-de/by-2-0", + "instruction": "Jede Nutzung ist unter den Bedingungen dieser \\u201eDatenlizenz Deutschland - Namensnennung - Version 2.0 zul\\u00e4ssig.\\nDie bereitgestellten Daten und Metadaten d\\u00fcrfen f\\u00fcr die kommerzielle und nicht kommerzielle Nutzung insbesondere:(1) vervielf\\u00e4ltigt, ausgedruckt, pr\\u00e4sentiert, ver\\u00e4ndert, bearbeitet sowie an Dritte \\u00fcbermittelt werden;\\n (2) mit eigenen Daten und Daten Anderer zusammengef\\u00fchrt und zu selbst\\u00e4ndigen neuen Datens\\u00e4tzen verbunden werden;\\n (3) in interne und externe Gesch\\u00e4ftsprozesse, Produkte und Anwendungen in \\u00f6ffentlichen und nicht \\u00f6ffentlichen elektronischen Netzwerken eingebunden werden.\\nBei der Nutzung ist sicherzustellen, dass folgende Angaben als Quellenvermerk enthalten sind:\\n(1) Bezeichnung des Bereitstellers nach dessen Ma\\u00dfgabe,\\n(2) der Vermerk Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 sowie\\n(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.\\nVer\\u00e4nderungen, Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk mit dem Hinweis zu versehen, dass die Daten ge\\u00e4ndert wurden.", + "attribution": "\\u00a9 Statistische \\u00c4mter des Bundes und der L\\u00e4nder 2014" + } + ] + }, + { + "title": "Dokumentation - Zensus 2011 - Methoden und Verfahren", + "description": "Diese Publikation beschreibt ausf\\u00fchrlich die Methoden und Verfahren des registergest\\u00fctzten Zensus 2011; von der Datengewinnung und -aufbereitung bis hin zur Ergebniserstellung und Geheimhaltung. Der vorliegende Band wurde von den Statistischen \\u00c4mtern des Bundes und der L\\u00e4nder im Juni 2015 ver\\u00f6ffentlicht.", + "path": "https://www.destatis.de/DE/Publikationen/Thematisch/Bevoelkerung/Zensus/ZensusBuLaMethodenVerfahren5121105119004.pdf?__blob=publicationFile", + "licenses": [ + { + "name": "dl-by-de/2.0", + "title": "Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0", + "path": "www.govdata.de/dl-de/by-2-0", + "instruction": "Jede Nutzung ist unter den Bedingungen dieser \\u201eDatenlizenz Deutschland - Namensnennung - Version 2.0 zul\\u00e4ssig.\\nDie bereitgestellten Daten und Metadaten d\\u00fcrfen f\\u00fcr die kommerzielle und nicht kommerzielle Nutzung insbesondere:(1) vervielf\\u00e4ltigt, ausgedruckt, pr\\u00e4sentiert, ver\\u00e4ndert, bearbeitet sowie an Dritte \\u00fcbermittelt werden;\\n (2) mit eigenen Daten und Daten Anderer zusammengef\\u00fchrt und zu selbst\\u00e4ndigen neuen Datens\\u00e4tzen verbunden werden;\\n (3) in interne und externe Gesch\\u00e4ftsprozesse, Produkte und Anwendungen in \\u00f6ffentlichen und nicht \\u00f6ffentlichen elektronischen Netzwerken eingebunden werden.\\nBei der Nutzung ist sicherzustellen, dass folgende Angaben als Quellenvermerk enthalten sind:\\n(1) Bezeichnung des Bereitstellers nach dessen Ma\\u00dfgabe,\\n(2) der Vermerk Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 sowie\\n(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.\\nVer\\u00e4nderungen, Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk mit dem Hinweis zu versehen, dass die Daten ge\\u00e4ndert wurden.", + "attribution": "\\u00a9 Statistisches Bundesamt, Wiesbaden 2015 (im Auftrag der Herausgebergemeinschaft)" + } + ] + }, + { + "title": "DemandRegio", + "description": "Harmonisierung und Entwicklung von Verfahren zur regionalen und zeitlichen Aufl\u00f6sung von Energienachfragen", + "path": "https://doi.org/10.34805/ffe-119-20", + "license": [ + { + "name": "Creative Commons Attribution 4.0 International", + "title": "CC BY 4.0", + "path": "https://creativecommons.org/licenses/by/4.0/", + "instruction": "You are free: To Share, To Adapt; As long as you: Attribute!", + "attribution": "\u00a9 FZJ, TUB, FfE" + } + ] + } + ], + "licenses": [ + { + "name": "ODbL-1.0", + "title": "Open Data Commons Open Database License 1.0", + "path": "https://opendatacommons.org/licenses/odbl/1.0/", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute, Share-Alike, Keep open!", + "attribution": "\u00a9 Reiner Lemoine Institut" + } + ], + "contributors": [ + { + "title": "nailend", + "email": "https://github.com/nailend", + "date": "2023-03-13", + "object": "metadata", + "comment": "Create metadata" + } + ], + "resources": [ + { + "profile": "tabular-data-resource", + "name": "demand.egon_etrago_electricity_households", + "path": "", + "format": "PostgreSQL", + "encoding": "UTF-8", + "schema": { + "fields": [ + { + "name": "bus_id", + "description": "MV bus_id (reference: demand.egon_cts_electricity_demand_building_share.bus_id, grid.egon_hvmv_substation.bus_id, grid.egon_mv_grid_district.bus_id))", + "type": "integer", + "unit": "none" + }, + { + "name": "scn_name", + "description": "Scenario name", + "type": "character varying", + "unit": "none" + }, + { + "name": "p_set", + "description": "Residential electricity demand load timeseries for one year (8760 hours) in MWh", + "type": "array of double precission", + "unit": "MWh" + } + ], + "primaryKey": "id, scn_name" + }, + "dialect": { + "delimiter": "", + "decimalSeparator": "." + } + } + ], + "review": { + "path": "", + "badge": "" + }, + "metaMetadata": { + "metadataVersion": "OEP-1.4.1", + "metadataLicense": { + "name": "CC0-1.0", + "title": "Creative Commons Zero v1.0 Universal", + "path": "https://creativecommons.org/publicdomain/zero/1.0/" + } + }, + "_comment": { + "metadata": "Metadata documentation and explanation (https://github.com/OpenEnergyPlatform/oemetadata/blob/master/metadata/v141/metadata_key_description.md)", + "dates": "Dates and time must follow the ISO8601 including time zone (YYYY-MM-DD or YYYY-MM-DDThh:mm:ss\u00b1hh)", + "units": "Use a space between numbers and units (100 m)", + "languages": "Languages must follow the IETF (BCP47) format (en-GB, en-US, de-DE)", + "licenses": "License name must follow the SPDX License List (https://spdx.org/licenses/)", + "review": "Following the OEP Data Review (https://github.com/OpenEnergyPlatform/data-preprocessing/wiki)", + "none": "If not applicable use (none)" + } +} diff --git a/src/egon/data/metadata/demand.egon_etrago_heat_cts.json b/src/egon/data/metadata/demand.egon_etrago_heat_cts.json new file mode 100644 index 000000000..6fff2bd10 --- /dev/null +++ b/src/egon/data/metadata/demand.egon_etrago_heat_cts.json @@ -0,0 +1,211 @@ +{ + "name": "demand.egon_etrago_heat_cts", + "title": "MV cts heat demand profiles", + "id": "WILL_BE_SET_AT_PUBLICATION", + "description": "Aggregated heat demand profile of cts sector at mvgd bus level for two scenarios. Values are given in MWh.", + "language": "undefined", + "keywords": [ + "" + ], + "publicationDate": "2023-03-13", + "context": { + "homepage": "https://ego-n.org/", + "documentation": "https://egon-data.readthedocs.io/en/latest/", + "sourceCode": "https://github.com/openego/eGon-data", + "contact": "https://ego-n.org/partners/", + "grantNo": "03EI1002", + "fundingAgency": "Bundesministerium f\u00fcr Wirtschaft und Klimaschutz", + "fundingAgencyLogo": "https://commons.wikimedia.org/wiki/File:BMWi_Logo_2021.svg#/media/File:BMWi_Logo_2021.svg", + "publisherLogo": "https://ego-n.org/images/eGon_logo_noborder_transbg.svg" + }, + "spatial": { + "location": "", + "extent": "Germany", + "resolution": "MV Grid" + }, + "temporal": { + "referenceDate": "2023-03-13", + "timeseries": { + "start": "2016-01-01", + "end": "2016-12-31", + "resolution": "1 h", + "alignment": "left", + "aggregationType": "sum" + } + }, + "sources": [ + { + "title": "Data bundle for egon-data: A transparent and reproducible data processing pipeline for energy system modeling", + "description": "egon-data provides a transparent and reproducible open data based data processing pipeline for generating data models suitable for energy system modeling. The data is customized for the requirements of the research project eGon. The research project aims to develop tools for an open and cross-sectoral planning of transmission and distribution grids. For further information please visit the eGon project website or its Github repository. egon-data retrieves and processes data from several different external input sources. As not all data dependencies can be downloaded automatically from external sources we provide a data bundle to be downloaded by egon-data.", + "path": "https://doi.org/10.5281/zenodo.6630616", + "licenses": [ + { + "name": "CC-BY-4.0", + "title": "Creative Commons Attribution 4.0 International", + "path": "https://creativecommons.org/licenses/by/4.0/legalcode", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute.", + "attribution": "\u00a9 eGon-data developers, \u00a9 Leibniz Institute for Applied Geophysics, \u00a9 Fraunhofer IEE, \u00a9 BGR Hannover, \u00a9 Danielle Schmidt, \u00a9 \u00dcbertragungsnetzbetreiber, \u00a9 DIW Berlin, \u00a9 Reiner Lemoine Institut, \u00a9 Statistisches Bundesamt, Wiesbaden 2008, \u00a9 Statistische \u00c4mter des Bundes und der L\u00e4nder 2021" + } + ] + }, + { + "title": "eGon-data", + "description": "Workflow to download, process and generate data sets suitable for the further research conducted in the project eGon (https://ego-n.org/)", + "path": "https://github.com/openego/eGon-data", + "licenses": [ + { + "name": "AGPL-3.0 License", + "title": "GNU Affero General Public License v3.0", + "path": "https://www.gnu.org/licenses/agpl-3.0.de.html", + "instruction": "Permissions of this strongest copyleft license are conditioned on making available complete source code of licensed works and modifications, which include larger works using a licensed work, under the same license. Copyright and license notices must be preserved. Contributors provide an express grant of patent rights. When a modified version is used to provide a service over a network, the complete source code of the modified version must be made available.", + "attribution": "\u00a9 eGon-data developers" + } + ] + }, + { + "title": "Verwaltungsgebiete 1:250 000 (Ebenen)", + "description": "Der Datenbestand umfasst s\u00e4mtliche Verwaltungseinheiten der hierarchischen Verwaltungsebenen vom Staat bis zu den Gemeinden mit ihren Grenzen, statistischen Schl\u00fcsselzahlen, Namen der Verwaltungseinheit sowie die spezifische Bezeichnung der Verwaltungsebene des jeweiligen Landes.", + "path": "https://daten.gdz.bkg.bund.de/produkte/vg/vg250_ebenen_0101/2020/vg250_01-01.geo84.shape.ebenen.zip", + "licenses": [ + { + "name": "dl-by-de/2.0", + "title": "Datenlizenz Deutschland \u2013 Namensnennung \u2013 Version 2.0", + "path": "www.govdata.de/dl-de/by-2-0", + "instruction": "Jede Nutzung ist unter den Bedingungen dieser \u201eDatenlizenz Deutschland - Namensnennung - Version 2.0 zul\u00e4ssig.Die bereitgestellten Daten und Metadaten d\u00fcrfen f\u00fcr die kommerzielle und nicht kommerzielle Nutzung insbesondere:(1) vervielf\u00e4ltigt, ausgedruckt, pr\u00e4sentiert, ver\u00e4ndert, bearbeitet sowie an Dritte \u00fcbermittelt werden; (2) mit eigenen Daten und Daten Anderer zusammengef\u00fchrt und zu selbst\u00e4ndigen neuen Datens\u00e4tzen verbunden werden; (3) in interne und externe Gesch\u00e4ftsprozesse, Produkte und Anwendungen in \u00f6ffentlichen und nicht \u00f6ffentlichen elektronischen Netzwerken eingebunden werden.Bei der Nutzung ist sicherzustellen, dass folgende Angaben als Quellenvermerk enthalten sind:(1) Bezeichnung des Bereitstellers nach dessen Ma\u00dfgabe,(2) der Vermerk Datenlizenz Deutschland \u2013 Namensnennung \u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 sowie(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.Ver\u00e4nderungen, Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk mit dem Hinweis zu versehen, dass die Daten ge\u00e4ndert wurden.", + "attribution": "\u00a9 Bundesamt f\u00fcr Kartographie und Geod\u00e4sie 2020 (Daten ver\u00e4ndert)" + } + ] + }, + { + "title": "Statistisches Bundesamt (Destatis) - Ergebnisse des Zensus 2011 zum Download", + "description": "Als Download bieten wir Ihnen auf dieser Seite zus\\u00e4tzlich zur Zensusdatenbank CSV- und teilweise Excel-Tabellen mit umfassenden Personen-, Haushalts- und Familien- sowie Geb\\u00e4ude- und Wohnungsmerkmalen. Die Ergebnisse liegen auf Bundes-, L\\u00e4nder-, Kreis- und Gemeindeebene vor. Au\\u00dferdem sind einzelne Ergebnisse f\\u00fcr Gitterzellen verf\\u00fcgbar.", + "path": "https://www.zensus2011.de/DE/Home/Aktuelles/DemografischeGrunddaten.html", + "licenses": [ + { + "name": "dl-by-de/2.0", + "title": "Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0", + "path": "www.govdata.de/dl-de/by-2-0", + "instruction": "Jede Nutzung ist unter den Bedingungen dieser \\u201eDatenlizenz Deutschland - Namensnennung - Version 2.0 zul\\u00e4ssig.\\nDie bereitgestellten Daten und Metadaten d\\u00fcrfen f\\u00fcr die kommerzielle und nicht kommerzielle Nutzung insbesondere:(1) vervielf\\u00e4ltigt, ausgedruckt, pr\\u00e4sentiert, ver\\u00e4ndert, bearbeitet sowie an Dritte \\u00fcbermittelt werden;\\n (2) mit eigenen Daten und Daten Anderer zusammengef\\u00fchrt und zu selbst\\u00e4ndigen neuen Datens\\u00e4tzen verbunden werden;\\n (3) in interne und externe Gesch\\u00e4ftsprozesse, Produkte und Anwendungen in \\u00f6ffentlichen und nicht \\u00f6ffentlichen elektronischen Netzwerken eingebunden werden.\\nBei der Nutzung ist sicherzustellen, dass folgende Angaben als Quellenvermerk enthalten sind:\\n(1) Bezeichnung des Bereitstellers nach dessen Ma\\u00dfgabe,\\n(2) der Vermerk Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 sowie\\n(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.\\nVer\\u00e4nderungen, Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk mit dem Hinweis zu versehen, dass die Daten ge\\u00e4ndert wurden.", + "attribution": "\\u00a9 Statistische \\u00c4mter des Bundes und der L\\u00e4nder 2014" + } + ] + }, + { + "title": "Dokumentation - Zensus 2011 - Methoden und Verfahren", + "description": "Diese Publikation beschreibt ausf\\u00fchrlich die Methoden und Verfahren des registergest\\u00fctzten Zensus 2011; von der Datengewinnung und -aufbereitung bis hin zur Ergebniserstellung und Geheimhaltung. Der vorliegende Band wurde von den Statistischen \\u00c4mtern des Bundes und der L\\u00e4nder im Juni 2015 ver\\u00f6ffentlicht.", + "path": "https://www.destatis.de/DE/Publikationen/Thematisch/Bevoelkerung/Zensus/ZensusBuLaMethodenVerfahren5121105119004.pdf?__blob=publicationFile", + "licenses": [ + { + "name": "dl-by-de/2.0", + "title": "Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0", + "path": "www.govdata.de/dl-de/by-2-0", + "instruction": "Jede Nutzung ist unter den Bedingungen dieser \\u201eDatenlizenz Deutschland - Namensnennung - Version 2.0 zul\\u00e4ssig.\\nDie bereitgestellten Daten und Metadaten d\\u00fcrfen f\\u00fcr die kommerzielle und nicht kommerzielle Nutzung insbesondere:(1) vervielf\\u00e4ltigt, ausgedruckt, pr\\u00e4sentiert, ver\\u00e4ndert, bearbeitet sowie an Dritte \\u00fcbermittelt werden;\\n (2) mit eigenen Daten und Daten Anderer zusammengef\\u00fchrt und zu selbst\\u00e4ndigen neuen Datens\\u00e4tzen verbunden werden;\\n (3) in interne und externe Gesch\\u00e4ftsprozesse, Produkte und Anwendungen in \\u00f6ffentlichen und nicht \\u00f6ffentlichen elektronischen Netzwerken eingebunden werden.\\nBei der Nutzung ist sicherzustellen, dass folgende Angaben als Quellenvermerk enthalten sind:\\n(1) Bezeichnung des Bereitstellers nach dessen Ma\\u00dfgabe,\\n(2) der Vermerk Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 sowie\\n(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.\\nVer\\u00e4nderungen, Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk mit dem Hinweis zu versehen, dass die Daten ge\\u00e4ndert wurden.", + "attribution": "\\u00a9 Statistisches Bundesamt, Wiesbaden 2015 (im Auftrag der Herausgebergemeinschaft)" + } + ] + }, + { + "title": "ERA5 global reanalysis", + "description": "ERA5 is the fifth generation ECMWF reanalysis for the global climate and weather for the past 4 to 7 decades. Currently data is available from 1950, split into Climate Data Store entries for 1950-1978 (preliminary back extension) and from 1979 onwards (final release plus timely updates, this page). ERA5 replaces the ERA-Interim reanalysis. See the online ERA5 documentation (https://confluence.ecmwf.int/display/CKB/ERA5%3A+data+documentation#ERA5:datadocumentation-Dataupdatefrequency) for more information.", + "path": "https://confluence.ecmwf.int/display/CKB/ERA5%3A+data+documentation#ERA5:datadocumentation-Dataupdatefrequency", + "licenses": [ + { + "name": "Licence to use Copernicus Products", + "title": "Licence to use Copernicus Products", + "path": "https://cds.climate.copernicus.eu/api/v2/terms/static/licence-to-use-copernicus-products.pdf", + "instruction": "This Licence is free of charge, worldwide, non-exclusive, royalty free and perpetual. Access to Copernicus Products is given for any purpose in so far as it is lawful, whereas use may include, but is not limited to: reproduction; distribution; communication to the public; adaptation, modification and combination with other data and information; or any combination of the foregoing", + "attribution": "\u00a9 Copernicus Climate Change Service (C3S) Climate Data Store" + } + ] + }, + { + "title": "Peta5 0 1 HD ser", + "description": "modelled Heat Demand distribution (in GJ per hectare grid cell) for service-sector heat demands for space heating, hot water and process heat for the year 2015 using HRE4 data and the combined top-down bottom-up approach of HRE4", + "path": "https://s-eenergies-open-data-euf.hub.arcgis.com/search", + "licenses": [ + { + "name": "Creative Commons Attribution 4.0 International", + "title": "CC BY 4.0", + "path": "https://creativecommons.org/licenses/by/4.0/", + "instruction": "You are free: To Share, To Adapt; As long as you: Attribute!", + "attribution": "\\u00a9 Flensburg, Halmstad and Aalborg universities" + } + ] + } + ], + "licenses": [ + { + "name": "ODbL-1.0", + "title": "Open Data Commons Open Database License 1.0", + "path": "https://opendatacommons.org/licenses/odbl/1.0/", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute, Share-Alike, Keep open!", + "attribution": "\u00a9 Reiner Lemoine Institut" + } + ], + "contributors": [ + { + "title": "nailend", + "email": "https://github.com/nailend", + "date": "2023-03-13", + "object": "metadata", + "comment": "Create metadata" + } + ], + "resources": [ + { + "profile": "tabular-data-resource", + "name": "demand.egon_etrago_heat_cts", + "path": "", + "format": "PostgreSQL", + "encoding": "UTF-8", + "schema": { + "fields": [ + { + "name": "bus_id", + "description": "MV bus_id (reference: demand.egon_cts_heat_demand_building_share.bus_id, grid.egon_hvmv_substation.bus_id, grid.egon_mv_grid_district.bus_id)", + "type": "integer", + "unit": "none" + }, + { + "name": "scn_name", + "description": "Scenario name (reference: demand.egon_cts_heat_demand_building_share.scenario)", + "type": "character varying", + "unit": "none" + }, + { + "name": "p_set", + "description": "Cts heat demand load timeseries for one year (8760 hours) in MWh", + "type": "array of real", + "unit": "MWh" + } + ], + "primaryKey": "id, scn_name" + }, + "dialect": { + "delimiter": "", + "decimalSeparator": "." + } + } + ], + "review": { + "path": "", + "badge": "" + }, + "metaMetadata": { + "metadataVersion": "OEP-1.4.1", + "metadataLicense": { + "name": "CC0-1.0", + "title": "Creative Commons Zero v1.0 Universal", + "path": "https://creativecommons.org/publicdomain/zero/1.0/" + } + }, + "_comment": { + "metadata": "Metadata documentation and explanation (https://github.com/OpenEnergyPlatform/oemetadata/blob/master/metadata/v141/metadata_key_description.md)", + "dates": "Dates and time must follow the ISO8601 including time zone (YYYY-MM-DD or YYYY-MM-DDThh:mm:ss\u00b1hh)", + "units": "Use a space between numbers and units (100 m)", + "languages": "Languages must follow the IETF (BCP47) format (en-GB, en-US, de-DE)", + "licenses": "License name must follow the SPDX License List (https://spdx.org/licenses/)", + "review": "Following the OEP Data Review (https://github.com/OpenEnergyPlatform/data-preprocessing/wiki)", + "none": "If not applicable use (none)" + } +} diff --git a/src/egon/data/metadata/demand.egon_heat_idp_pool.json b/src/egon/data/metadata/demand.egon_heat_idp_pool.json new file mode 100644 index 000000000..5f597c7f8 --- /dev/null +++ b/src/egon/data/metadata/demand.egon_heat_idp_pool.json @@ -0,0 +1,152 @@ +{ + "name": "demand.egon_heat_idp_pool", + "title": "eGon pool of residential intra-day heat demand profiles", + "id": "WILL_BE_SET_AT_PUBLICATION", + "description": "Pool of individual, normalized intra-day heat demand profiles of residential buildings. Timeseries only represent one day but are used to be merged for a whole year ", + "language": "undefined", + "keywords": [], + "publicationDate": "2022-09-14", + "context": { + "homepage": "https://ego-n.org/", + "documentation": "https://egon-data.readthedocs.io/en/latest/", + "sourceCode": "https://github.com/openego/eGon-data", + "contact": "https://ego-n.org/partners/", + "grantNo": "03EI1002", + "fundingAgency": "Bundesministerium für Wirtschaft und Klimaschutz", + "fundingAgencyLogo": "https://www.innovation-beratung-foerderung.de/INNO/Redaktion/DE/Bilder/Titelbilder/titel_foerderlogo_bmwi.jpg?__blob=normal&v=3", + "publisherLogo": "https://ego-n.org/images/eGon_logo_noborder_transbg.svg" + }, + "spatial": { + "location": "none", + "extent": "Germany", + "resolution": "" + }, + "temporal": { + "referenceDate": "2022-09-14", + "timeseries": { + "start": "2011-01-01 00:00", + "end": "2011-12-31 23:00", + "resolution": "1 h", + "alignment": "left", + "aggregationType": "sum" + } + }, + "sources": [ + { + "title": "eGon-data", + "description": "Workflow to download, process and generate data sets suitable for the further research conducted in the project eGon (https://ego-n.org/)", + "path": "https://github.com/openego/eGon-data", + "licenses": [ + { + "name": "AGPL-3.0 License", + "title": "GNU Affero General Public License v3.0", + "path": "https://www.gnu.org/licenses/agpl-3.0.de.html", + "instruction": "Permissions of this strongest copyleft license are conditioned on making available complete source code of licensed works and modifications, which include larger works using a licensed work, under the same license. Copyright and license notices must be preserved. Contributors provide an express grant of patent rights. When a modified version is used to provide a service over a network, the complete source code of the modified version must be made available.", + "attribution": "© eGon-data developers" + } + ] + }, + { + "title": "Data bundle for egon-data: A transparent and reproducible data processing pipeline for energy system modeling", + "description": "egon-data provides a transparent and reproducible open data based data processing pipeline for generating data models suitable for energy system modeling. The data is customized for the requirements of the research project eGon. The research project aims to develop tools for an open and cross-sectoral planning of transmission and distribution grids. For further information please visit the eGon project website or its Github repository. egon-data retrieves and processes data from several different external input sources. As not all data dependencies can be downloaded automatically from external sources we provide a data bundle to be downloaded by egon-data.", + "path": "https://doi.org/10.5281/zenodo.6630616", + "licenses": [ + { + "name": "CC-BY-4.0", + "title": "Creative Commons Attribution 4.0 International", + "path": "https://creativecommons.org/licenses/by/4.0/legalcode", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute.", + "attribution": "© eGon-data developers, © Leibniz Institute for Applied Geophysics, © Fraunhofer IEE, © BGR Hannover, © Danielle Schmidt, © Übertragungsnetzbetreiber, © DIW Berlin, © Reiner Lemoine Institut, © Statistisches Bundesamt, Wiesbaden 2008, © Statistische Ämter des Bundes und der Länder 2021" + } + ] + } + ], + "licenses": [ + { + "name": "CC-BY-4.0", + "title": "Creative Commons Attribution 4.0 International", + "path": "https://creativecommons.org/licenses/by/4.0/legalcode", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute.", + "attribution": "Europa-Universität Flensburg" + } + ], + "contributors": [ + { + "title": "ClaraBuettner", + "email": "https://github.com/ClaraBuettner", + "date": "2022-09-12", + "object": "metadata", + "comment": "Create metadata" + }, + { + "title": "nesnoj", + "email": "https://github.com/nesnoj", + "date": "2022-09-14", + "object": "metadata", + "comment": "Update metadata" + } + ], + "resources": [ + { + "profile": "tabular-data-resource", + "name": "demand.egon_heat_idp_pool", + "path": "", + "format": "PostgreSQL", + "encoding": "UTF-8", + "schema": { + "fields": [ + { + "name": "house", + "description": "Type of residential house, either SFH (single family house) or MFH (multi family house)", + "type": "str", + "unit": "none" + }, + { + "name": "idp", + "description": "Normalized intra-day-profile", + "type": "array of double precision", + "unit": "per unit" + }, + { + "name": "index", + "description": "Unique identifier", + "type": "integer", + "unit": "none" + }, + { + "name": "temperature_class", + "description": "Temperature class (considering the mean outside temeperature during the day)", + "type": "integer", + "unit": "none" + } + ], + "primaryKey": "index" + }, + "dialect": { + "delimiter": "", + "decimalSeparator": "" + } + } + ], + "review": { + "path": "", + "badge": "" + }, + "metaMetadata": { + "metadataVersion": "OEP-1.4.1", + "metadataLicense": { + "name": "CC0-1.0", + "title": "Creative Commons Zero v1.0 Universal", + "path": "https://creativecommons.org/publicdomain/zero/1.0/" + } + }, + "_comment": { + "metadata": "Metadata documentation and explanation (https://github.com/OpenEnergyPlatform/oemetadata/blob/master/metadata/v141/metadata_key_description.md)", + "dates": "Dates and time must follow the ISO8601 including time zone (YYYY-MM-DD or YYYY-MM-DDThh:mm:ss±hh)", + "units": "Use a space between numbers and units (100 m)", + "languages": "Languages must follow the IETF (BCP47) format (en-GB, en-US, de-DE)", + "licenses": "License name must follow the SPDX License List (https://spdx.org/licenses/)", + "review": "Following the OEP Data Review (https://github.com/OpenEnergyPlatform/data-preprocessing/wiki)", + "none": "If not applicable use (none)" + } +} diff --git a/src/egon/data/metadata/demand.egon_heat_timeseries_selected_profiles.json b/src/egon/data/metadata/demand.egon_heat_timeseries_selected_profiles.json new file mode 100644 index 000000000..ff1b05742 --- /dev/null +++ b/src/egon/data/metadata/demand.egon_heat_timeseries_selected_profiles.json @@ -0,0 +1,157 @@ +{ + "name": "demand.egon_heat_timeseries_selected_profiles", + "title": "eGon selected intra-day profiles per building", + "id": "WILL_BE_SET_AT_PUBLICATION", + "description": "List of selected intra-day-profiles for each residential building in Germany", + "language": [ + "en-EN", + "de-DE" + ], + "keywords": [], + "publicationDate": "2022-09-14", + "context": { + "homepage": "https://ego-n.org/", + "documentation": "https://egon-data.readthedocs.io/en/latest/", + "sourceCode": "https://github.com/openego/eGon-data", + "contact": "https://ego-n.org/partners/", + "grantNo": "03EI1002", + "fundingAgency": "Bundesministerium f\u00fcr Wirtschaft und Klimaschutz", + "fundingAgencyLogo": "https://www.innovation-beratung-foerderung.de/INNO/Redaktion/DE/Bilder/Titelbilder/titel_foerderlogo_bmwi.jpg?__blob=normal&v=3", + "publisherLogo": "https://ego-n.org/images/eGon_logo_noborder_transbg.svg" + }, + "spatial": { + "location": "none", + "extent": "Germany", + "resolution": "buildings" + }, + "temporal": { + "referenceDate": "2022-09-14", + "timeseries": {} + }, + "sources": [ + { + "title": "Data bundle for egon-data: A transparent and reproducible data processing pipeline for energy system modeling", + "description": "egon-data provides a transparent and reproducible open data based data processing pipeline for generating data models suitable for energy system modeling. The data is customized for the requirements of the research project eGon. The research project aims to develop tools for an open and cross-sectoral planning of transmission and distribution grids. For further information please visit the eGon project website or its Github repository. egon-data retrieves and processes data from several different external input sources. As not all data dependencies can be downloaded automatically from external sources we provide a data bundle to be downloaded by egon-data.", + "path": "https://doi.org/10.5281/zenodo.6630616", + "licenses": [ + { + "name": "CC-BY-4.0", + "title": "Creative Commons Attribution 4.0 International", + "path": "https://creativecommons.org/licenses/by/4.0/legalcode", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute.", + "attribution": "\u00a9 eGon-data developers, \u00a9 Leibniz Institute for Applied Geophysics, \u00a9 Fraunhofer IEE, \u00a9 BGR Hannover, \u00a9 Danielle Schmidt, \u00a9 \u00dcbertragungsnetzbetreiber, \u00a9 DIW Berlin, \u00a9 Reiner Lemoine Institut, \u00a9 Statistisches Bundesamt, Wiesbaden 2008, \u00a9 Statistische \u00c4mter des Bundes und der L\u00e4nder 2021" + } + ] + }, + { + "title": "eGon-data", + "description": "Workflow to download, process and generate data sets suitable for the further research conducted in the project eGon (https://ego-n.org/)", + "path": "https://github.com/openego/eGon-data", + "licenses": [ + { + "name": "AGPL-3.0 License", + "title": "GNU Affero General Public License v3.0", + "path": "https://www.gnu.org/licenses/agpl-3.0.de.html", + "instruction": "Permissions of this strongest copyleft license are conditioned on making available complete source code of licensed works and modifications, which include larger works using a licensed work, under the same license. Copyright and license notices must be preserved. Contributors provide an express grant of patent rights. When a modified version is used to provide a service over a network, the complete source code of the modified version must be made available.", + "attribution": "\u00a9 eGon-data developers" + } + ] + }, + { + "title": "OpenStreetMap Data Extracts (Geofabrik)", + "description": "Full data extract of OpenStreetMap data for defined spatial extent at ''referenceDate''", + "path": "https://download.geofabrik.de/europe/germany-210101.osm.pbf", + "licenses": [ + { + "name": "ODbL-1.0", + "title": "Open Data Commons Open Database License 1.0", + "path": "https://opendatacommons.org/licenses/odbl/1.0/", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute, Share-Alike, Keep open!", + "attribution": "\u00a9 OpenStreetMap contributors" + } + ] + } + ], + "licenses": [ + { + "name": "ODbL-1.0", + "title": "Open Data Commons Open Database License 1.0", + "path": "https://opendatacommons.org/licenses/odbl/1.0/", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute, Share-Alike, Keep open!", + "attribution": "\u00a9 Europa-Universit\u00e4t Flensburg" + } + ], + "contributors": [ + { + "title": "ClaraBuettner", + "email": "https://github.com/ClaraBuettner", + "date": "2022-09-12", + "object": "metadata", + "comment": "Create metadata" + }, + { + "title": "nesnoj", + "email": "https://github.com/nesnoj", + "date": "2022-09-14", + "object": "metadata", + "comment": "Update metadata" + } + ], + "resources": [ + { + "profile": "tabular-data-resource", + "name": "demand.egon_heat_timeseries_selected_profiles", + "path": "", + "format": "PostgreSQL", + "encoding": "UTF-8", + "schema": { + "fields": [ + { + "name": "building_id", + "description": "Index of the corresponding building", + "type": "integer", + "unit": "none" + }, + { + "name": "selected_idp_profiles", + "description": "List of selected profiles for each day in the year", + "type": "array of integer", + "unit": "none" + }, + { + "name": "zensus_population_id", + "description": "Index of corresponding census cell", + "type": "integer", + "unit": "none" + } + ], + "primaryKey": "building_id" + }, + "dialect": { + "delimiter": "", + "decimalSeparator": "" + } + } + ], + "review": { + "path": "", + "badge": "" + }, + "metaMetadata": { + "metadataVersion": "OEP-1.4.1", + "metadataLicense": { + "name": "CC0-1.0", + "title": "Creative Commons Zero v1.0 Universal", + "path": "https://creativecommons.org/publicdomain/zero/1.0/" + } + }, + "_comment": { + "metadata": "Metadata documentation and explanation (https://github.com/OpenEnergyPlatform/oemetadata/blob/master/metadata/v141/metadata_key_description.md)", + "dates": "Dates and time must follow the ISO8601 including time zone (YYYY-MM-DD or YYYY-MM-DDThh:mm:ss\u00b1hh)", + "units": "Use a space between numbers and units (100 m)", + "languages": "Languages must follow the IETF (BCP47) format (en-GB, en-US, de-DE)", + "licenses": "License name must follow the SPDX License List (https://spdx.org/licenses/)", + "review": "Following the OEP Data Review (https://github.com/OpenEnergyPlatform/data-preprocessing/wiki)", + "none": "If not applicable use (none)" + } +} diff --git a/src/egon/data/metadata/demand.egon_household_electricity_profile_in_census_cell.json b/src/egon/data/metadata/demand.egon_household_electricity_profile_in_census_cell.json new file mode 100644 index 000000000..3e0696b84 --- /dev/null +++ b/src/egon/data/metadata/demand.egon_household_electricity_profile_in_census_cell.json @@ -0,0 +1,218 @@ +{ + "name": "demand.egon_household_electricity_profile_in_census_cell", + "title": "eGon household electricity profiles in census cells", + "id": "WILL_BE_SET_AT_PUBLICATION", + "description": "Mapping table for residential electricity profiles to census cell including scaling factors for two scenarios (eGon2035, eGon100RE).", + "language": [ + "en-EN", + "de-DE" + ], + "keywords": [ + "" + ], + "publicationDate": "2022-09-14", + "context": { + "homepage": "https://ego-n.org/", + "documentation": "https://egon-data.readthedocs.io/en/latest/", + "sourceCode": "https://github.com/openego/eGon-data", + "contact": "https://ego-n.org/partners/", + "grantNo": "03EI1002", + "fundingAgency": "Bundesministerium f\u00fcr Wirtschaft und Klimaschutz", + "fundingAgencyLogo": "https://commons.wikimedia.org/wiki/File:BMWi_Logo_2021.svg#/media/File:BMWi_Logo_2021.svg", + "publisherLogo": "https://ego-n.org/images/eGon_logo_noborder_transbg.svg" + }, + "spatial": { + "location": "", + "extent": "Germany", + "resolution": "100 m x 100 m" + }, + "temporal": { + "referenceDate": "2022-09-14", + "timeseries": {} + }, + "sources": [ + { + "title": "Data bundle for egon-data: A transparent and reproducible data processing pipeline for energy system modeling", + "description": "egon-data provides a transparent and reproducible open data based data processing pipeline for generating data models suitable for energy system modeling. The data is customized for the requirements of the research project eGon. The research project aims to develop tools for an open and cross-sectoral planning of transmission and distribution grids. For further information please visit the eGon project website or its Github repository. egon-data retrieves and processes data from several different external input sources. As not all data dependencies can be downloaded automatically from external sources we provide a data bundle to be downloaded by egon-data.", + "path": "https://doi.org/10.5281/zenodo.6630616", + "licenses": [ + { + "name": "CC-BY-4.0", + "title": "Creative Commons Attribution 4.0 International", + "path": "https://creativecommons.org/licenses/by/4.0/legalcode", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute.", + "attribution": "\u00a9 eGon-data developers, \u00a9 Leibniz Institute for Applied Geophysics, \u00a9 Fraunhofer IEE, \u00a9 BGR Hannover, \u00a9 Danielle Schmidt, \u00a9 \u00dcbertragungsnetzbetreiber, \u00a9 DIW Berlin, \u00a9 Reiner Lemoine Institut, \u00a9 Statistisches Bundesamt, Wiesbaden 2008, \u00a9 Statistische \u00c4mter des Bundes und der L\u00e4nder 2021" + } + ] + }, + { + "title": "eGon-data", + "description": "Workflow to download, process and generate data sets suitable for the further research conducted in the project eGon (https://ego-n.org/)", + "path": "https://github.com/openego/eGon-data", + "licenses": [ + { + "name": "AGPL-3.0 License", + "title": "GNU Affero General Public License v3.0", + "path": "https://www.gnu.org/licenses/agpl-3.0.de.html", + "instruction": "Permissions of this strongest copyleft license are conditioned on making available complete source code of licensed works and modifications, which include larger works using a licensed work, under the same license. Copyright and license notices must be preserved. Contributors provide an express grant of patent rights. When a modified version is used to provide a service over a network, the complete source code of the modified version must be made available.", + "attribution": "\u00a9 eGon-data developers" + } + ] + }, + { + "title": "Verwaltungsgebiete 1:250 000 (Ebenen)", + "description": "Der Datenbestand umfasst s\u00e4mtliche Verwaltungseinheiten der hierarchischen Verwaltungsebenen vom Staat bis zu den Gemeinden mit ihren Grenzen, statistischen Schl\u00fcsselzahlen, Namen der Verwaltungseinheit sowie die spezifische Bezeichnung der Verwaltungsebene des jeweiligen Landes.", + "path": "https://daten.gdz.bkg.bund.de/produkte/vg/vg250_ebenen_0101/2020/vg250_01-01.geo84.shape.ebenen.zip", + "licenses": [ + { + "name": "dl-by-de/2.0", + "title": "Datenlizenz Deutschland \u2013 Namensnennung \u2013 Version 2.0", + "path": "www.govdata.de/dl-de/by-2-0", + "instruction": "Jede Nutzung ist unter den Bedingungen dieser \u201eDatenlizenz Deutschland - Namensnennung - Version 2.0 zul\u00e4ssig.Die bereitgestellten Daten und Metadaten d\u00fcrfen f\u00fcr die kommerzielle und nicht kommerzielle Nutzung insbesondere:(1) vervielf\u00e4ltigt, ausgedruckt, pr\u00e4sentiert, ver\u00e4ndert, bearbeitet sowie an Dritte \u00fcbermittelt werden; (2) mit eigenen Daten und Daten Anderer zusammengef\u00fchrt und zu selbst\u00e4ndigen neuen Datens\u00e4tzen verbunden werden; (3) in interne und externe Gesch\u00e4ftsprozesse, Produkte und Anwendungen in \u00f6ffentlichen und nicht \u00f6ffentlichen elektronischen Netzwerken eingebunden werden.Bei der Nutzung ist sicherzustellen, dass folgende Angaben als Quellenvermerk enthalten sind:(1) Bezeichnung des Bereitstellers nach dessen Ma\u00dfgabe,(2) der Vermerk Datenlizenz Deutschland \u2013 Namensnennung \u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 sowie(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.Ver\u00e4nderungen, Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk mit dem Hinweis zu versehen, dass die Daten ge\u00e4ndert wurden.", + "attribution": "\u00a9 Bundesamt f\u00fcr Kartographie und Geod\u00e4sie 2020 (Daten ver\u00e4ndert)" + } + ] + }, + { + "title": "OpenStreetMap Data Extracts (Geofabrik)", + "description": "Full data extract of OpenStreetMap data for defined spatial extent at 2022-01-01", + "path": "https://download.geofabrik.de/europe/germany-210101.osm.pbf", + "licenses": [ + { + "name": "ODbL-1.0", + "title": "Open Data Commons Open Database License 1.0", + "path": "https://opendatacommons.org/licenses/odbl/1.0/", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute, Share-Alike, Keep open!", + "attribution": "\u00a9 OpenStreetMap contributors" + } + ] + }, + { + "title": "DemandRegio", + "description": "Harmonisierung und Entwicklung von Verfahren zur regionalen und zeitlichen Aufl\u00f6sung von Energienachfragen", + "path": "https://doi.org/10.34805/ffe-119-20", + "license": [ + { + "name": "Creative Commons Attribution 4.0 International", + "title": "CC BY 4.0", + "path": "https://creativecommons.org/licenses/by/4.0/", + "instruction": "You are free: To Share, To Adapt; As long as you: Attribute!", + "attribution": "\u00a9 FZJ, TUB, FfE" + } + ] + } + ], + "licenses": [ + { + "name": "ODbL-1.0", + "title": "Open Data Commons Open Database License 1.0", + "path": "https://opendatacommons.org/licenses/odbl/1.0/", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute, Share-Alike, Keep open!", + "attribution": "\u00a9 Reiner Lemoine Institut" + } + ], + "contributors": [ + { + "title": "ClaraBuettner", + "email": "https://github.com/ClaraBuettner", + "date": "2022-09-12", + "object": "metadata", + "comment": "Create metadata" + }, + { + "title": "nailend", + "email": "https://github.com/nailend", + "date": "2022-09-12", + "object": "metadata", + "comment": "Updated metadata" + }, + { + "title": "nesnoj", + "email": "https://github.com/nesnoj", + "date": "2022-09-14", + "object": "metadata", + "comment": "Update metadata" + } + ], + "resources": [ + { + "profile": "tabular-data-resource", + "name": "demand.egon_household_electricity_profile_in_census_cell", + "path": "", + "format": "PostgreSQL", + "encoding": "UTF-8", + "schema": { + "fields": [ + { + "name": "cell_id", + "description": "Census cell id (reference to: society.destatis_zensus_population_per_ha_inside_germany.id)", + "type": "integer", + "unit": "none" + }, + { + "name": "grid_id", + "description": "Descriptive ID of census cell given by Statistisches Bundesamt (Destatis)", + "type": "character varying", + "unit": "none" + }, + { + "name": "cell_profile_ids", + "description": "Household profile ids", + "type": "array of integer", + "unit": "none" + }, + { + "name": "nuts3", + "description": "NUTS-3 ID of the census cell", + "type": "character varying", + "unit": "none" + }, + { + "name": "nuts1", + "description": "NUTS-1 ID of the census cell", + "type": "character varying", + "unit": "none" + }, + { + "name": "factor_2035", + "description": "Scaling factor for all profiles in the respective census cell for scenario eGon2035", + "type": "double precision", + "unit": "none" + }, + { + "name": "factor_2050", + "description": "Scaling factor for all profiles in the respective census cell for scenario eGon100RE", + "type": "double precision", + "unit": "none" + } + ], + "primaryKey": "cell_id" + }, + "dialect": { + "delimiter": "", + "decimalSeparator": "." + } + } + ], + "review": { + "path": "", + "badge": "" + }, + "metaMetadata": { + "metadataVersion": "OEP-1.4.1", + "metadataLicense": { + "name": "CC0-1.0", + "title": "Creative Commons Zero v1.0 Universal", + "path": "https://creativecommons.org/publicdomain/zero/1.0/" + } + }, + "_comment": { + "metadata": "Metadata documentation and explanation (https://github.com/OpenEnergyPlatform/oemetadata/blob/master/metadata/v141/metadata_key_description.md)", + "dates": "Dates and time must follow the ISO8601 including time zone (YYYY-MM-DD or YYYY-MM-DDThh:mm:ss\u00b1hh)", + "units": "Use a space between numbers and units (100 m)", + "languages": "Languages must follow the IETF (BCP47) format (en-GB, en-US, de-DE)", + "licenses": "License name must follow the SPDX License List (https://spdx.org/licenses/)", + "review": "Following the OEP Data Review (https://github.com/OpenEnergyPlatform/data-preprocessing/wiki)", + "none": "If not applicable use (none)" + } +} diff --git a/src/egon/data/metadata/demand.egon_household_electricity_profile_of_buildings.json b/src/egon/data/metadata/demand.egon_household_electricity_profile_of_buildings.json new file mode 100644 index 000000000..f302d0a27 --- /dev/null +++ b/src/egon/data/metadata/demand.egon_household_electricity_profile_of_buildings.json @@ -0,0 +1,170 @@ +{ + "name": "demand.egon_household_electricity_profile_of_buildings", + "title": "eGon household electricity profiles of buildings", + "id": "WILL_BE_SET_AT_PUBLICATION", + "description": "Mapping table for residential electricity profiles to buildings via project internal building_id and corresponding census cell_id. ", + "language": [ + "en-EN", + "de-DE" + ], + "keywords": [], + "publicationDate": "2022-09-14", + "context": { + "homepage": "https://ego-n.org/", + "documentation": "https://egon-data.readthedocs.io/en/latest/", + "sourceCode": "https://github.com/openego/eGon-data", + "contact": "https://ego-n.org/partners/", + "grantNo": "03EI1002", + "fundingAgency": "Bundesministerium f\u00fcr Wirtschaft und Klimaschutz", + "fundingAgencyLogo": "https://commons.wikimedia.org/wiki/File:BMWi_Logo_2021.svg#/media/File:BMWi_Logo_2021.svg", + "publisherLogo": "https://ego-n.org/images/eGon_logo_noborder_transbg.svg" + }, + "spatial": { + "location": "", + "extent": "Germany", + "resolution": "Buildings" + }, + "temporal": { + "referenceDate": "2022-09-14", + "timeseries": {} + }, + "sources": [ + { + "title": "Data bundle for egon-data: A transparent and reproducible data processing pipeline for energy system modeling", + "description": "egon-data provides a transparent and reproducible open data based data processing pipeline for generating data models suitable for energy system modeling. The data is customized for the requirements of the research project eGon. The research project aims to develop tools for an open and cross-sectoral planning of transmission and distribution grids. For further information please visit the eGon project website or its Github repository. egon-data retrieves and processes data from several different external input sources. As not all data dependencies can be downloaded automatically from external sources we provide a data bundle to be downloaded by egon-data.", + "path": "https://doi.org/10.5281/zenodo.6630616", + "licenses": [ + { + "name": "CC-BY-4.0", + "title": "Creative Commons Attribution 4.0 International", + "path": "https://creativecommons.org/licenses/by/4.0/legalcode", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute.", + "attribution": "\u00a9 eGon-data developers, \u00a9 Leibniz Institute for Applied Geophysics, \u00a9 Fraunhofer IEE, \u00a9 BGR Hannover, \u00a9 Danielle Schmidt, \u00a9 \u00dcbertragungsnetzbetreiber, \u00a9 DIW Berlin, \u00a9 Reiner Lemoine Institut, \u00a9 Statistisches Bundesamt, Wiesbaden 2008, \u00a9 Statistische \u00c4mter des Bundes und der L\u00e4nder 2021" + } + ] + }, + { + "title": "eGon-data", + "description": "Workflow to download, process and generate data sets suitable for the further research conducted in the project eGon (https://ego-n.org/)", + "path": "https://github.com/openego/eGon-data", + "licenses": [ + { + "name": "AGPL-3.0 License", + "title": "GNU Affero General Public License v3.0", + "path": "https://www.gnu.org/licenses/agpl-3.0.de.html", + "instruction": "Permissions of this strongest copyleft license are conditioned on making available complete source code of licensed works and modifications, which include larger works using a licensed work, under the same license. Copyright and license notices must be preserved. Contributors provide an express grant of patent rights. When a modified version is used to provide a service over a network, the complete source code of the modified version must be made available.", + "attribution": "\u00a9 eGon-data developers" + } + ] + }, + { + "title": "OpenStreetMap Data Extracts (Geofabrik)", + "description": "Full data extract of OpenStreetMap data for defined spatial extent at 2022-01-01", + "path": "https://download.geofabrik.de/europe/germany-210101.osm.pbf", + "licenses": [ + { + "name": "ODbL-1.0", + "title": "Open Data Commons Open Database License 1.0", + "path": "https://opendatacommons.org/licenses/odbl/1.0/", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute, Share-Alike, Keep open!", + "attribution": "\u00a9 OpenStreetMap contributors" + } + ] + } + ], + "licenses": [ + { + "name": "ODbL-1.0", + "title": "Open Data Commons Open Database License 1.0", + "path": "https://opendatacommons.org/licenses/odbl/1.0/", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute, Share-Alike, Keep open!", + "attribution": "\u00a9 Reiner Lemoine Institut" + } + ], + "contributors": [ + { + "title": "ClaraBuettner", + "email": "https://github.com/ClaraBuettner", + "date": "2022-09-12", + "object": "metadata", + "comment": "Create metadata" + }, + { + "title": "nailend", + "email": "https://github.com/nailend", + "date": "2022-09-12", + "object": "metadata", + "comment": "Updated metadata" + }, + { + "title": "nesnoj", + "email": "https://github.com/nesnoj", + "date": "2022-09-14", + "object": "metadata", + "comment": "Update metadata" + } + ], + "resources": [ + { + "profile": "tabular-data-resource", + "name": "demand.egon_household_electricity_profile_of_buildings", + "path": "", + "format": "PostgreSQL", + "encoding": "UTF-8", + "schema": { + "fields": [ + { + "name": "id", + "description": "Unique identifier", + "type": "integer", + "unit": "none" + }, + { + "name": "building_id", + "description": "Building id (reference to: openstreetmap.osm_buildings_residential.id, openstreetmap.osm_buildings_synthetic.id)", + "type": "integer", + "unit": "none" + }, + { + "name": "cell_id", + "description": "Census cell id (reference to: society.destatis_zensus_population_per_ha_inside_germany.id)", + "type": "integer", + "unit": "none" + }, + { + "name": "profile_id", + "description": "ID of iee household profiles (reference to: iee_household_load_profiles.type)", + "type": "character varying", + "unit": "none" + } + ], + "primaryKey": "id" + }, + "dialect": { + "delimiter": "", + "decimalSeparator": "." + } + } + ], + "review": { + "path": "", + "badge": "" + }, + "metaMetadata": { + "metadataVersion": "OEP-1.4.1", + "metadataLicense": { + "name": "CC0-1.0", + "title": "Creative Commons Zero v1.0 Universal", + "path": "https://creativecommons.org/publicdomain/zero/1.0/" + } + }, + "_comment": { + "metadata": "Metadata documentation and explanation (https://github.com/OpenEnergyPlatform/oemetadata/blob/master/metadata/v141/metadata_key_description.md)", + "dates": "Dates and time must follow the ISO8601 including time zone (YYYY-MM-DD or YYYY-MM-DDThh:mm:ss\u00b1hh)", + "units": "Use a space between numbers and units (100 m)", + "languages": "Languages must follow the IETF (BCP47) format (en-GB, en-US, de-DE)", + "licenses": "License name must follow the SPDX License List (https://spdx.org/licenses/)", + "review": "Following the OEP Data Review (https://github.com/OpenEnergyPlatform/data-preprocessing/wiki)", + "none": "If not applicable use (none)" + } +} diff --git a/src/egon/data/metadata/demand.iee_household_load_profiles.json b/src/egon/data/metadata/demand.iee_household_load_profiles.json new file mode 100644 index 000000000..093fc47dd --- /dev/null +++ b/src/egon/data/metadata/demand.iee_household_load_profiles.json @@ -0,0 +1,158 @@ +{ + "name": "demand.iee_household_load_profiles", + "title": "eGon pool of electricity household load profiles", + "id": "WILL_BE_SET_AT_PUBLICATION", + "description": "100.000 annual profiles in hourly resolution of electricity demand of private households for different household types (singles, couples, other) with varying number of elderly and children. The profiles were created using a bottom-up load profile generator by Fraunhofer IEE developed in the Bachelors thesis \"Auswirkungen verschiedener Haushaltslastprofile auf PV-Batterie-Systeme\" by Jonas Haack, Fachhochschule Flensburg, December 2012. The columns are named as follows: \"a\", e.g. P2a0000 is the first profile of a couples household with 2 children. See publication below for the list of prefixes. Values are given in Wh.", + "language": [ + "en-EN", + "de-DE" + ], + "keywords": [ + "" + ], + "publicationDate": "2022-09-14", + "context": { + "homepage": "https://ego-n.org/", + "documentation": "https://egon-data.readthedocs.io/en/latest/", + "sourceCode": "https://github.com/openego/eGon-data", + "contact": "https://ego-n.org/partners/", + "grantNo": "03EI1002", + "fundingAgency": "Bundesministerium f\u00fcr Wirtschaft und Klimaschutz", + "fundingAgencyLogo": "https://commons.wikimedia.org/wiki/File:BMWi_Logo_2021.svg#/media/File:BMWi_Logo_2021.svg", + "publisherLogo": "https://ego-n.org/images/eGon_logo_noborder_transbg.svg" + }, + "spatial": { + "location": "", + "extent": "Germany", + "resolution": "" + }, + "temporal": { + "referenceDate": "2022-06-10", + "timeseries": { + "start": "2016-01-01", + "end": "2016-12-31", + "resolution": "1 h", + "alignment": "left", + "aggregationType": "sum" + } + }, + "sources": [ + { + "title": "Data bundle for egon-data: A transparent and reproducible data processing pipeline for energy system modeling", + "description": "egon-data provides a transparent and reproducible open data based data processing pipeline for generating data models suitable for energy system modeling. The data is customized for the requirements of the research project eGon. The research project aims to develop tools for an open and cross-sectoral planning of transmission and distribution grids. For further information please visit the eGon project website or its Github repository. egon-data retrieves and processes data from several different external input sources. As not all data dependencies can be downloaded automatically from external sources we provide a data bundle to be downloaded by egon-data.", + "path": "https://doi.org/10.5281/zenodo.6630616", + "licenses": [ + { + "name": "CC-BY-4.0", + "title": "Creative Commons Attribution 4.0 International", + "path": "https://creativecommons.org/licenses/by/4.0/legalcode", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute.", + "attribution": "\u00a9 eGon-data developers, \u00a9 Leibniz Institute for Applied Geophysics, \u00a9 Fraunhofer IEE, \u00a9 BGR Hannover, \u00a9 Danielle Schmidt, \u00a9 \u00dcbertragungsnetzbetreiber, \u00a9 DIW Berlin, \u00a9 Reiner Lemoine Institut, \u00a9 Statistisches Bundesamt, Wiesbaden 2008, \u00a9 Statistische \u00c4mter des Bundes und der L\u00e4nder 2021" + } + ] + }, + { + "title": "eGon-data", + "description": "Workflow to download, process and generate data sets suitable for the further research conducted in the project eGon (https://ego-n.org/)", + "path": "https://github.com/openego/eGon-data", + "licenses": [ + { + "name": "AGPL-3.0 License", + "title": "GNU Affero General Public License v3.0", + "path": "https://www.gnu.org/licenses/agpl-3.0.de.html", + "instruction": "Permissions of this strongest copyleft license are conditioned on making available complete source code of licensed works and modifications, which include larger works using a licensed work, under the same license. Copyright and license notices must be preserved. Contributors provide an express grant of patent rights. When a modified version is used to provide a service over a network, the complete source code of the modified version must be made available.", + "attribution": "\u00a9 eGon-data developers" + } + ] + } + ], + "licenses": [ + { + "name": "ODbL-1.0", + "title": "Open Data Commons Open Database License 1.0", + "path": "https://opendatacommons.org/licenses/odbl/1.0/", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute, Share-Alike, Keep open!", + "attribution": "\u00a9 Reiner Lemoine Institut" + } + ], + "contributors": [ + { + "title": "ClaraBuettner", + "email": "https://github.com/ClaraBuettner", + "date": "2022-09-12", + "object": "metadata", + "comment": "Create metadata" + }, + { + "title": "nailend", + "email": "https://github.com/nailend", + "date": "2022-09-12", + "object": "metadata", + "comment": "Updated metadata" + }, + { + "title": "nesnoj", + "email": "https://github.com/nesnoj", + "date": "2022-09-14", + "object": "metadata", + "comment": "Update metadata" + } + ], + "resources": [ + { + "profile": "tabular-data-resource", + "name": "demand.iee_household_load_profiles", + "path": "", + "format": "PostgreSQL", + "encoding": "UTF-8", + "schema": { + "fields": [ + { + "name": "id", + "description": "Unique identifier", + "type": "integer", + "unit": "none" + }, + { + "name": "type", + "description": "Profile id", + "type": "character(8)", + "unit": "none" + }, + { + "name": "load_in_wh", + "description": "Residential demand load timeseries for one year (8760 hours) in Wh", + "type": "array of real", + "unit": "Wh" + } + ], + "primaryKey": "id" + }, + "dialect": { + "delimiter": "", + "decimalSeparator": "." + } + } + ], + "review": { + "path": "", + "badge": "" + }, + "metaMetadata": { + "metadataVersion": "OEP-1.4.1", + "metadataLicense": { + "name": "CC0-1.0", + "title": "Creative Commons Zero v1.0 Universal", + "path": "https://creativecommons.org/publicdomain/zero/1.0/" + } + }, + "_comment": { + "metadata": "Metadata documentation and explanation (https://github.com/OpenEnergyPlatform/oemetadata/blob/master/metadata/v141/metadata_key_description.md)", + "dates": "Dates and time must follow the ISO8601 including time zone (YYYY-MM-DD or YYYY-MM-DDThh:mm:ss\u00b1hh)", + "units": "Use a space between numbers and units (100 m)", + "languages": "Languages must follow the IETF (BCP47) format (en-GB, en-US, de-DE)", + "licenses": "License name must follow the SPDX License List (https://spdx.org/licenses/)", + "review": "Following the OEP Data Review (https://github.com/OpenEnergyPlatform/data-preprocessing/wiki)", + "none": "If not applicable use (none)" + } +} diff --git a/src/egon/data/metadata/grid.egon_mv_grid_district.json b/src/egon/data/metadata/grid.egon_mv_grid_district.json new file mode 100644 index 000000000..a8eb56501 --- /dev/null +++ b/src/egon/data/metadata/grid.egon_mv_grid_district.json @@ -0,0 +1,150 @@ +{ + "name": "grid.egon_mv_grid_district", + "title": "eGon MV grid districts", + "id": "WILL_BE_SET_AT_PUBLICATION", + "description": "Mapping table of bus_id to polygons shapes for MV grid districts.", + "language": "undefined", + "keywords": [], + "publicationDate": "2023-03-13", + "context": { + "homepage": "https://ego-n.org/", + "documentation": "https://egon-data.readthedocs.io/en/latest/", + "sourceCode": "https://github.com/openego/eGon-data", + "contact": "https://ego-n.org/partners/", + "grantNo": "03EI1002", + "fundingAgency": "Bundesministerium f\u00fcr Wirtschaft und Klimaschutz", + "fundingAgencyLogo": "https://commons.wikimedia.org/wiki/File:BMWi_Logo_2021.svg#/media/File:BMWi_Logo_2021.svg", + "publisherLogo": "https://ego-n.org/images/eGon_logo_noborder_transbg.svg" + }, + "spatial": { + "location": "", + "extent": "Germany", + "resolution": "MV Grids" + }, + "temporal": {}, + "sources": [ + { + "title": "Data bundle for egon-data: A transparent and reproducible data processing pipeline for energy system modeling", + "description": "egon-data provides a transparent and reproducible open data based data processing pipeline for generating data models suitable for energy system modeling. The data is customized for the requirements of the research project eGon. The research project aims to develop tools for an open and cross-sectoral planning of transmission and distribution grids. For further information please visit the eGon project website or its Github repository. egon-data retrieves and processes data from several different external input sources. As not all data dependencies can be downloaded automatically from external sources we provide a data bundle to be downloaded by egon-data.", + "path": "https://doi.org/10.5281/zenodo.6630616", + "licenses": [ + { + "name": "CC-BY-4.0", + "title": "Creative Commons Attribution 4.0 International", + "path": "https://creativecommons.org/licenses/by/4.0/legalcode", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute.", + "attribution": "\u00a9 eGon-data developers, \u00a9 Leibniz Institute for Applied Geophysics, \u00a9 Fraunhofer IEE, \u00a9 BGR Hannover, \u00a9 Danielle Schmidt, \u00a9 \u00dcbertragungsnetzbetreiber, \u00a9 DIW Berlin, \u00a9 Reiner Lemoine Institut, \u00a9 Statistisches Bundesamt, Wiesbaden 2008, \u00a9 Statistische \u00c4mter des Bundes und der L\u00e4nder 2021" + } + ] + }, + { + "title": "eGon-data", + "description": "Workflow to download, process and generate data sets suitable for the further research conducted in the project eGon (https://ego-n.org/)", + "path": "https://github.com/openego/eGon-data", + "licenses": [ + { + "name": "AGPL-3.0 License", + "title": "GNU Affero General Public License v3.0", + "path": "https://www.gnu.org/licenses/agpl-3.0.de.html", + "instruction": "Permissions of this strongest copyleft license are conditioned on making available complete source code of licensed works and modifications, which include larger works using a licensed work, under the same license. Copyright and license notices must be preserved. Contributors provide an express grant of patent rights. When a modified version is used to provide a service over a network, the complete source code of the modified version must be made available.", + "attribution": "\u00a9 eGon-data developers" + } + ] + }, + { + "title": "Verwaltungsgebiete 1:250 000 (Ebenen)", + "description": "Der Datenbestand umfasst s\u00e4mtliche Verwaltungseinheiten der hierarchischen Verwaltungsebenen vom Staat bis zu den Gemeinden mit ihren Grenzen, statistischen Schl\u00fcsselzahlen, Namen der Verwaltungseinheit sowie die spezifische Bezeichnung der Verwaltungsebene des jeweiligen Landes.", + "path": "https://daten.gdz.bkg.bund.de/produkte/vg/vg250_ebenen_0101/2020/vg250_01-01.geo84.shape.ebenen.zip", + "licenses": [ + { + "name": "dl-by-de/2.0", + "title": "Datenlizenz Deutschland \u2013 Namensnennung \u2013 Version 2.0", + "path": "www.govdata.de/dl-de/by-2-0", + "instruction": "Jede Nutzung ist unter den Bedingungen dieser \u201eDatenlizenz Deutschland - Namensnennung - Version 2.0 zul\u00e4ssig.Die bereitgestellten Daten und Metadaten d\u00fcrfen f\u00fcr die kommerzielle und nicht kommerzielle Nutzung insbesondere:(1) vervielf\u00e4ltigt, ausgedruckt, pr\u00e4sentiert, ver\u00e4ndert, bearbeitet sowie an Dritte \u00fcbermittelt werden; (2) mit eigenen Daten und Daten Anderer zusammengef\u00fchrt und zu selbst\u00e4ndigen neuen Datens\u00e4tzen verbunden werden; (3) in interne und externe Gesch\u00e4ftsprozesse, Produkte und Anwendungen in \u00f6ffentlichen und nicht \u00f6ffentlichen elektronischen Netzwerken eingebunden werden.Bei der Nutzung ist sicherzustellen, dass folgende Angaben als Quellenvermerk enthalten sind:(1) Bezeichnung des Bereitstellers nach dessen Ma\u00dfgabe,(2) der Vermerk Datenlizenz Deutschland \u2013 Namensnennung \u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 sowie(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.Ver\u00e4nderungen, Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk mit dem Hinweis zu versehen, dass die Daten ge\u00e4ndert wurden.", + "attribution": "\u00a9 Bundesamt f\u00fcr Kartographie und Geod\u00e4sie 2020 (Daten ver\u00e4ndert)" + } + ] + } + ], + "licenses": [ + { + "name": "ODbL-1.0", + "title": "Open Data Commons Open Database License 1.0", + "path": "https://opendatacommons.org/licenses/odbl/1.0/", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute, Share-Alike, Keep open!", + "attribution": "\u00a9 Reiner Lemoine Institut, \u00a9 Bundesamt f\u00fcr Kartographie und Geod\u00e4sie 2020 (Daten ver\u00e4ndert), \u00a9 eGon-data developers" + } + ], + "contributors": [ + { + "title": "nailend", + "email": "https://github.com/nailend", + "date": "2023-03-13", + "object": "metadata", + "comment": "create metadata" + } + ], + "resources": [ + { + "profile": "tabular-data-resource", + "name": "grid.egon_mv_grid_district", + "path": "", + "format": "PostgreSQL", + "encoding": "UTF-8", + "schema": { + "fields": [ + { + "name": "building_id", + "description": "Building id (reference to: openstreetmap.osm_buildings_filtered.id, openstreetmap.osm_buildings_synthetic.id)", + "type": "integer", + "unit": "none" + }, + { + "name": "bus_id", + "description": "MVGD bus id (reference to: grid.egon_hvmv_substation.bus_id, grid.egon_mv_grid_district.bus_id)", + "type": "integer", + "unit": "none" + }, + { + "name": "geom", + "description": "Polygon of mv grid district (CRS: ERTS89/EPSG:3035)", + "type": "geometry", + "unit": "None" + }, + { + "name": "area", + "description": "Area covered by mvgd", + "type": "double precision", + "unit": "m^2" + } + ], + "primaryKey": "bus_id" + }, + "dialect": { + "delimiter": "", + "decimalSeparator": "." + } + } + ], + "review": { + "path": "", + "badge": "" + }, + "metaMetadata": { + "metadataVersion": "OEP-1.4.1", + "metadataLicense": { + "name": "CC0-1.0", + "title": "Creative Commons Zero v1.0 Universal", + "path": "https://creativecommons.org/publicdomain/zero/1.0/" + } + }, + "_comment": { + "metadata": "Metadata documentation and explanation (https://github.com/OpenEnergyPlatform/oemetadata/blob/master/metadata/v141/metadata_key_description.md)", + "dates": "Dates and time must follow the ISO8601 including time zone (YYYY-MM-DD or YYYY-MM-DDThh:mm:ss\u00b1hh)", + "units": "Use a space between numbers and units (100 m)", + "languages": "Languages must follow the IETF (BCP47) format (en-GB, en-US, de-DE)", + "licenses": "License name must follow the SPDX License List (https://spdx.org/licenses/)", + "review": "Following the OEP Data Review (https://github.com/OpenEnergyPlatform/data-preprocessing/wiki)", + "none": "If not applicable use (none)" + } +} diff --git a/src/egon/data/metadata/openstreetmap.osm_buildings_filtered.json b/src/egon/data/metadata/openstreetmap.osm_buildings_filtered.json new file mode 100644 index 000000000..519c87941 --- /dev/null +++ b/src/egon/data/metadata/openstreetmap.osm_buildings_filtered.json @@ -0,0 +1,183 @@ +{ + "name": "openstreetmap.osm_buildings_filtered", + "title": "eGon filtered buildings from openstreetmap", + "id": "WILL_BE_SET_AT_PUBLICATION", + "description": "Tag filtered list of buildings from OpenStreetMap which might have electricity or heat demand - (c) OpenStreetMap contributors", + "language": "undefined", + "keywords": [], + "publicationDate": "2023-03-13", + "context": { + "homepage": "https://ego-n.org/", + "documentation": "https://egon-data.readthedocs.io/en/latest/", + "sourceCode": "https://github.com/openego/eGon-data", + "contact": "https://ego-n.org/partners/", + "grantNo": "03EI1002", + "fundingAgency": "Bundesministerium f\u00fcr Wirtschaft und Klimaschutz", + "fundingAgencyLogo": "https://commons.wikimedia.org/wiki/File:BMWi_Logo_2021.svg#/media/File:BMWi_Logo_2021.svg", + "publisherLogo": "https://ego-n.org/images/eGon_logo_noborder_transbg.svg" + }, + "spatial": { + "location": "", + "extent": "Germany", + "resolution": "Buildings" + }, + "temporal": { + "referenceDate": "2022-01-01", + "timeseries": {} + }, + "sources": [ + { + "title": "Data bundle for egon-data: A transparent and reproducible data processing pipeline for energy system modeling", + "description": "egon-data provides a transparent and reproducible open data based data processing pipeline for generating data models suitable for energy system modeling. The data is customized for the requirements of the research project eGon. The research project aims to develop tools for an open and cross-sectoral planning of transmission and distribution grids. For further information please visit the eGon project website or its Github repository. egon-data retrieves and processes data from several different external input sources. As not all data dependencies can be downloaded automatically from external sources we provide a data bundle to be downloaded by egon-data.", + "path": "https://doi.org/10.5281/zenodo.6630616", + "licenses": [ + { + "name": "CC-BY-4.0", + "title": "Creative Commons Attribution 4.0 International", + "path": "https://creativecommons.org/licenses/by/4.0/legalcode", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute.", + "attribution": "\u00a9 eGon-data developers, \u00a9 Leibniz Institute for Applied Geophysics, \u00a9 Fraunhofer IEE, \u00a9 BGR Hannover, \u00a9 Danielle Schmidt, \u00a9 \u00dcbertragungsnetzbetreiber, \u00a9 DIW Berlin, \u00a9 Reiner Lemoine Institut, \u00a9 Statistisches Bundesamt, Wiesbaden 2008, \u00a9 Statistische \u00c4mter des Bundes und der L\u00e4nder 2021" + } + ] + }, + { + "title": "eGon-data", + "description": "Workflow to download, process and generate data sets suitable for the further research conducted in the project eGon (https://ego-n.org/)", + "path": "https://github.com/openego/eGon-data", + "licenses": [ + { + "name": "AGPL-3.0 License", + "title": "GNU Affero General Public License v3.0", + "path": "https://www.gnu.org/licenses/agpl-3.0.de.html", + "instruction": "Permissions of this strongest copyleft license are conditioned on making available complete source code of licensed works and modifications, which include larger works using a licensed work, under the same license. Copyright and license notices must be preserved. Contributors provide an express grant of patent rights. When a modified version is used to provide a service over a network, the complete source code of the modified version must be made available.", + "attribution": "\u00a9 eGon-data developers" + } + ] + }, + { + "title": "OpenStreetMap Data Extracts (Geofabrik)", + "description": "Full data extract of OpenStreetMap data for defined spatial extent at 2022-01-01", + "path": "https://download.geofabrik.de/europe/germany-210101.osm.pbf", + "licenses": [ + { + "name": "ODbL-1.0", + "title": "Open Data Commons Open Database License 1.0", + "path": "https://opendatacommons.org/licenses/odbl/1.0/", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute, Share-Alike, Keep open!", + "attribution": "\u00a9 OpenStreetMap contributors" + } + ] + } + ], + "licenses": [ + { + "name": "ODbL-1.0", + "title": "Open Data Commons Open Database License 1.0", + "path": "https://opendatacommons.org/licenses/odbl/1.0/", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute, Share-Alike, Keep open!", + "attribution": "\u00a9 Reiner Lemoine Institut" + } + ], + "contributors": [ + { + "title": "nailend", + "email": "https://github.com/nailend", + "date": "2023-03-13", + "object": "metadata", + "comment": "Created metadata" + } + ], + "resources": [ + { + "profile": "tabular-data-resource", + "name": "openstreetmap.osm_buildings_filtered", + "path": "", + "format": "PostgreSQL", + "encoding": "UTF-8", + "schema": { + "fields": [ + { + "name": "osm_id", + "description": "Openstreetmap identifier", + "type": "bigint", + "unit": "none" + }, + { + "name": "amenitiy", + "description": "Type of amenitiy if given", + "type": "text", + "unit": "none" + }, + { + "name": "building", + "description": "Type of building", + "type": "text", + "unit": "none" + }, + { + "name": "name", + "description": "Name of the building if given", + "type": "text", + "unit": "none" + }, + { + "name": "geom_building", + "description": "Polygon of the building (CRS: ERTS89/EPSG:3035)", + "type": "geometry", + "unit": "none" + }, + { + "name": "area", + "description": "Surface area of building", + "type": "double precision", + "unit": "m^2" + }, + { + "name": "geom_point", + "description": "Centroid of the building (CRS: ERTS89/EPSG:3035)", + "type": "geometry", + "unit": "none" + }, + { + "name": "tags", + "description": "Opensteetmap tags assigned to the building", + "type": "hstore", + "unit": "none" + }, + { + "name": "id", + "description": "Unique identifyer and project internal building id (reference to: demand.egon_hp_capacity_buildings.building_id, demand.egon_cts_heat_demand_building_share.building_id, demand.egon_cts_electricity_demand_building_share.building_id, demand.egon_household_electricity_profile_of_buildings.building_id, demand.egon_building_heat_peak_loads.building_id, demand.egon_building_electricity_peak_loads.building_id, boundaries.egon_map_zensus_mvgd_buildings.building_id)", + "type": "integer", + "unit": "none" + } + ], + "primaryKey": "id" + }, + "dialect": { + "delimiter": "", + "decimalSeparator": "." + } + } + ], + "review": { + "path": "", + "badge": "" + }, + "metaMetadata": { + "metadataVersion": "OEP-1.4.1", + "metadataLicense": { + "name": "CC0-1.0", + "title": "Creative Commons Zero v1.0 Universal", + "path": "https://creativecommons.org/publicdomain/zero/1.0/" + } + }, + "_comment": { + "metadata": "Metadata documentation and explanation (https://github.com/OpenEnergyPlatform/oemetadata/blob/master/metadata/v141/metadata_key_description.md)", + "dates": "Dates and time must follow the ISO8601 including time zone (YYYY-MM-DD or YYYY-MM-DDThh:mm:ss\u00b1hh)", + "units": "Use a space between numbers and units (100 m)", + "languages": "Languages must follow the IETF (BCP47) format (en-GB, en-US, de-DE)", + "licenses": "License name must follow the SPDX License List (https://spdx.org/licenses/)", + "review": "Following the OEP Data Review (https://github.com/OpenEnergyPlatform/data-preprocessing/wiki)", + "none": "If not applicable use (none)" + } +} diff --git a/src/egon/data/metadata/openstreetmap.osm_buildings_synthetic.json b/src/egon/data/metadata/openstreetmap.osm_buildings_synthetic.json new file mode 100644 index 000000000..5957f61f3 --- /dev/null +++ b/src/egon/data/metadata/openstreetmap.osm_buildings_synthetic.json @@ -0,0 +1,188 @@ +{ + "name": "openstreetmap.osm_buildings_synthetic", + "title": "eGon synthetic residential buildings", + "id": "WILL_BE_SET_AT_PUBLICATION", + "description": " List of generated synthetic buildings ", + "language": [ + "en-EN", + "de-DE" + ], + "keywords": [], + "publicationDate": "2022-09-14", + "context": { + "homepage": "https://ego-n.org/", + "documentation": "https://egon-data.readthedocs.io/en/latest/", + "sourceCode": "https://github.com/openego/eGon-data", + "contact": "https://ego-n.org/partners/", + "grantNo": "03EI1002", + "fundingAgency": "Bundesministerium f\u00fcr Wirtschaft und Klimaschutz", + "fundingAgencyLogo": "https://commons.wikimedia.org/wiki/File:BMWi_Logo_2021.svg#/media/File:BMWi_Logo_2021.svg", + "publisherLogo": "https://ego-n.org/images/eGon_logo_noborder_transbg.svg" + }, + "spatial": { + "location": "", + "extent": "Germany", + "resolution": "Buildings" + }, + "temporal": { + "referenceDate": "2022-09-14", + "timeseries": {} + }, + "sources": [ + { + "title": "Data bundle for egon-data: A transparent and reproducible data processing pipeline for energy system modeling", + "description": "egon-data provides a transparent and reproducible open data based data processing pipeline for generating data models suitable for energy system modeling. The data is customized for the requirements of the research project eGon. The research project aims to develop tools for an open and cross-sectoral planning of transmission and distribution grids. For further information please visit the eGon project website or its Github repository. egon-data retrieves and processes data from several different external input sources. As not all data dependencies can be downloaded automatically from external sources we provide a data bundle to be downloaded by egon-data.", + "path": "https://doi.org/10.5281/zenodo.6630616", + "licenses": [ + { + "name": "CC-BY-4.0", + "title": "Creative Commons Attribution 4.0 International", + "path": "https://creativecommons.org/licenses/by/4.0/legalcode", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute.", + "attribution": "\u00a9 eGon-data developers, \u00a9 Leibniz Institute for Applied Geophysics, \u00a9 Fraunhofer IEE, \u00a9 BGR Hannover, \u00a9 Danielle Schmidt, \u00a9 \u00dcbertragungsnetzbetreiber, \u00a9 DIW Berlin, \u00a9 Reiner Lemoine Institut, \u00a9 Statistisches Bundesamt, Wiesbaden 2008, \u00a9 Statistische \u00c4mter des Bundes und der L\u00e4nder 2021" + } + ] + }, + { + "title": "eGon-data", + "description": "Workflow to download, process and generate data sets suitable for the further research conducted in the project eGon (https://ego-n.org/)", + "path": "https://github.com/openego/eGon-data", + "licenses": [ + { + "name": "AGPL-3.0 License", + "title": "GNU Affero General Public License v3.0", + "path": "https://www.gnu.org/licenses/agpl-3.0.de.html", + "instruction": "Permissions of this strongest copyleft license are conditioned on making available complete source code of licensed works and modifications, which include larger works using a licensed work, under the same license. Copyright and license notices must be preserved. Contributors provide an express grant of patent rights. When a modified version is used to provide a service over a network, the complete source code of the modified version must be made available.", + "attribution": "\u00a9 eGon-data developers" + } + ] + }, + { + "title": "OpenStreetMap Data Extracts (Geofabrik)", + "description": "Full data extract of OpenStreetMap data for defined spatial extent at 2022-01-01", + "path": "https://download.geofabrik.de/europe/germany-210101.osm.pbf", + "licenses": [ + { + "name": "ODbL-1.0", + "title": "Open Data Commons Open Database License 1.0", + "path": "https://opendatacommons.org/licenses/odbl/1.0/", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute, Share-Alike, Keep open!", + "attribution": "\u00a9 OpenStreetMap contributors" + } + ] + } + ], + "licenses": [ + { + "name": "ODbL-1.0", + "title": "Open Data Commons Open Database License 1.0", + "path": "https://opendatacommons.org/licenses/odbl/1.0/", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute, Share-Alike, Keep open!", + "attribution": "\u00a9 Reiner Lemoine Institut" + } + ], + "contributors": [ + { + "title": "ClaraBuettner", + "email": "https://github.com/ClaraBuettner", + "date": "2022-09-12", + "object": "metadata", + "comment": "Create metadata" + }, + { + "title": "nailend", + "email": "https://github.com/nailend", + "date": "2022-09-12", + "object": "metadata", + "comment": "Updated metadata" + }, + { + "title": "nesnoj", + "email": "https://github.com/nesnoj", + "date": "2022-09-14", + "object": "metadata", + "comment": "Update metadata" + } + ], + "resources": [ + { + "profile": "tabular-data-resource", + "name": "openstreetmap.osm_buildings_synthetic", + "path": "", + "format": "PostgreSQL", + "encoding": "UTF-8", + "schema": { + "fields": [ + { + "name": "id", + "description": "Unique identifyer and project internal building id (reference to: openstreetmap.osm_buildings_residential.id, demand.egon_household_electricity_profile_of_buildings.building_id)", + "type": "character varying", + "unit": "none" + }, + { + "name": "area", + "description": "Surface area of building", + "type": "real", + "unit": "m^2" + }, + { + "name": "building", + "description": "Type of building (residential or cts)", + "type": "character varying(11)", + "unit": "none" + }, + { + "name": "cell_id", + "description": "Census cell id (reference to: society.destatis_zensus_population_per_ha_inside_germany.id)", + "type": "character varying", + "unit": "none" + }, + { + "name": "geom_building", + "description": "Polygon of building (CRS: ERTS89/EPSG:3035)", + "type": "geometry", + "unit": "none" + }, + { + "name": "geom_point", + "description": "Centroid of building (CRS: ERTS89/EPSG:3035)", + "type": "geometry", + "unit": "none" + }, + { + "name": "n_amenities_inside", + "description": "Number of amenities inside the building (always zero if not cts building)", + "type": "integer", + "unit": "none" + } + ], + "primaryKey": "id" + }, + "dialect": { + "delimiter": "", + "decimalSeparator": "." + } + } + ], + "review": { + "path": "", + "badge": "" + }, + "metaMetadata": { + "metadataVersion": "OEP-1.4.1", + "metadataLicense": { + "name": "CC0-1.0", + "title": "Creative Commons Zero v1.0 Universal", + "path": "https://creativecommons.org/publicdomain/zero/1.0/" + } + }, + "_comment": { + "metadata": "Metadata documentation and explanation (https://github.com/OpenEnergyPlatform/oemetadata/blob/master/metadata/v141/metadata_key_description.md)", + "dates": "Dates and time must follow the ISO8601 including time zone (YYYY-MM-DD or YYYY-MM-DDThh:mm:ss\u00b1hh)", + "units": "Use a space between numbers and units (100 m)", + "languages": "Languages must follow the IETF (BCP47) format (en-GB, en-US, de-DE)", + "licenses": "License name must follow the SPDX License List (https://spdx.org/licenses/)", + "review": "Following the OEP Data Review (https://github.com/OpenEnergyPlatform/data-preprocessing/wiki)", + "none": "If not applicable use (none)" + } +} diff --git a/src/egon/data/metadata/society.destatis_zensus_population_per_ha.json b/src/egon/data/metadata/society.destatis_zensus_population_per_ha.json new file mode 100644 index 000000000..5e20057a6 --- /dev/null +++ b/src/egon/data/metadata/society.destatis_zensus_population_per_ha.json @@ -0,0 +1,142 @@ +{ + "name": "society.destatis_zensus_population_per_ha", + "title": "DESTATIS - Zensus 2011 - Population per hectar", + "id": "WILL_BE_SET_AT_PUBLICATION", + "description": "National census in Germany in 2011.", + "language": "undefined", + "keywords": [], + "publicationDate": "2022-08-06", + "context": { + "homepage": "https://ego-n.org/", + "documentation": "https://egon-data.readthedocs.io/en/latest/", + "sourceCode": "https://github.com/openego/eGon-data", + "contact": "https://ego-n.org/partners/", + "grantNo": "03EI1002", + "fundingAgency": "Bundesministerium f\\u00fcr Wirtschaft und Energie", + "fundingAgencyLogo": "https://www.innovation-beratung-foerderung.de/INNO/Redaktion/DE/Bilder/Titelbilder/titel_foerderlogo_bmwi.jpg?__blob=normal&v=3", + "publisherLogo": "https://ego-n.org/images/eGon_logo_noborder_transbg.svg" + }, + "spatial": { + "location": "", + "extent": "Germany", + "resolution": "1 ha" + }, + "temporal": {}, + "sources": [ + { + "title": "Statistisches Bundesamt (Destatis) - Ergebnisse des Zensus 2011 zum Download", + "description": "Als Download bieten wir Ihnen auf dieser Seite zus\\u00e4tzlich zur Zensusdatenbank CSV- und teilweise Excel-Tabellen mit umfassenden Personen-, Haushalts- und Familien- sowie Geb\\u00e4ude- und Wohnungsmerkmalen. Die Ergebnisse liegen auf Bundes-, L\\u00e4nder-, Kreis- und Gemeindeebene vor. Au\\u00dferdem sind einzelne Ergebnisse f\\u00fcr Gitterzellen verf\\u00fcgbar.", + "path": "https://www.zensus2011.de/DE/Home/Aktuelles/DemografischeGrunddaten.html", + "licenses": [ + { + "name": "dl-by-de/2.0", + "title": "Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0", + "path": "www.govdata.de/dl-de/by-2-0", + "instruction": "Jede Nutzung ist unter den Bedingungen dieser \\u201eDatenlizenz Deutschland - Namensnennung - Version 2.0 zul\\u00e4ssig.\\nDie bereitgestellten Daten und Metadaten d\\u00fcrfen f\\u00fcr die kommerzielle und nicht kommerzielle Nutzung insbesondere:(1) vervielf\\u00e4ltigt, ausgedruckt, pr\\u00e4sentiert, ver\\u00e4ndert, bearbeitet sowie an Dritte \\u00fcbermittelt werden;\\n (2) mit eigenen Daten und Daten Anderer zusammengef\\u00fchrt und zu selbst\\u00e4ndigen neuen Datens\\u00e4tzen verbunden werden;\\n (3) in interne und externe Gesch\\u00e4ftsprozesse, Produkte und Anwendungen in \\u00f6ffentlichen und nicht \\u00f6ffentlichen elektronischen Netzwerken eingebunden werden.\\nBei der Nutzung ist sicherzustellen, dass folgende Angaben als Quellenvermerk enthalten sind:\\n(1) Bezeichnung des Bereitstellers nach dessen Ma\\u00dfgabe,\\n(2) der Vermerk Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 sowie\\n(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.\\nVer\\u00e4nderungen, Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk mit dem Hinweis zu versehen, dass die Daten ge\\u00e4ndert wurden.", + "attribution": "\\u00a9 Statistische \\u00c4mter des Bundes und der L\\u00e4nder 2014" + } + ] + }, + { + "title": "Dokumentation - Zensus 2011 - Methoden und Verfahren", + "description": "Diese Publikation beschreibt ausf\\u00fchrlich die Methoden und Verfahren des registergest\\u00fctzten Zensus 2011; von der Datengewinnung und -aufbereitung bis hin zur Ergebniserstellung und Geheimhaltung. Der vorliegende Band wurde von den Statistischen \\u00c4mtern des Bundes und der L\\u00e4nder im Juni 2015 ver\\u00f6ffentlicht.", + "path": "https://www.destatis.de/DE/Publikationen/Thematisch/Bevoelkerung/Zensus/ZensusBuLaMethodenVerfahren5121105119004.pdf?__blob=publicationFile", + "licenses": [ + { + "name": "dl-by-de/2.0", + "title": "Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0", + "path": "www.govdata.de/dl-de/by-2-0", + "instruction": "Jede Nutzung ist unter den Bedingungen dieser \\u201eDatenlizenz Deutschland - Namensnennung - Version 2.0 zul\\u00e4ssig.\\nDie bereitgestellten Daten und Metadaten d\\u00fcrfen f\\u00fcr die kommerzielle und nicht kommerzielle Nutzung insbesondere:(1) vervielf\\u00e4ltigt, ausgedruckt, pr\\u00e4sentiert, ver\\u00e4ndert, bearbeitet sowie an Dritte \\u00fcbermittelt werden;\\n (2) mit eigenen Daten und Daten Anderer zusammengef\\u00fchrt und zu selbst\\u00e4ndigen neuen Datens\\u00e4tzen verbunden werden;\\n (3) in interne und externe Gesch\\u00e4ftsprozesse, Produkte und Anwendungen in \\u00f6ffentlichen und nicht \\u00f6ffentlichen elektronischen Netzwerken eingebunden werden.\\nBei der Nutzung ist sicherzustellen, dass folgende Angaben als Quellenvermerk enthalten sind:\\n(1) Bezeichnung des Bereitstellers nach dessen Ma\\u00dfgabe,\\n(2) der Vermerk Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 sowie\\n(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.\\nVer\\u00e4nderungen, Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk mit dem Hinweis zu versehen, dass die Daten ge\\u00e4ndert wurden.", + "attribution": "\\u00a9 Statistisches Bundesamt, Wiesbaden 2015 (im Auftrag der Herausgebergemeinschaft)" + } + ] + } + ], + "licenses": [ + { + "name": "dl-by-de/2.0", + "title": "Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0", + "path": "www.govdata.de/dl-de/by-2-0", + "instruction": "Jede Nutzung ist unter den Bedingungen dieser \\u201eDatenlizenz Deutschland - Namensnennung - Version 2.0 zul\\u00e4ssig.\\nDie bereitgestellten Daten und Metadaten d\\u00fcrfen f\\u00fcr die kommerzielle und nicht kommerzielle Nutzung insbesondere:(1) vervielf\\u00e4ltigt, ausgedruckt, pr\\u00e4sentiert, ver\\u00e4ndert, bearbeitet sowie an Dritte \\u00fcbermittelt werden;\\n (2) mit eigenen Daten und Daten Anderer zusammengef\\u00fchrt und zu selbst\\u00e4ndigen neuen Datens\\u00e4tzen verbunden werden;\\n (3) in interne und externe Gesch\\u00e4ftsprozesse, Produkte und Anwendungen in \\u00f6ffentlichen und nicht \\u00f6ffentlichen elektronischen Netzwerken eingebunden werden.\\nBei der Nutzung ist sicherzustellen, dass folgende Angaben als Quellenvermerk enthalten sind:\\n(1) Bezeichnung des Bereitstellers nach dessen Ma\\u00dfgabe,\\n(2) der Vermerk Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 sowie\\n(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.\\nVer\\u00e4nderungen, Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk mit dem Hinweis zu versehen, dass die Daten ge\\u00e4ndert wurden.", + "attribution": "\\u00a9 Statistische \\u00c4mter des Bundes und der L\\u00e4nder 2014; \\u00a9 Statistisches Bundesamt, Wiesbaden 2015 (Daten ver\\u00e4ndert)" + } + ], + "contributors": [ + { + "title": "nailend", + "email": "https://github.com/nailend", + "date": "2023-03-13", + "object": "metadata", + "comment": "Create metadata" + } + ], + "resources": [ + { + "profile": "tabular-data-resource", + "name": "society.destatis_zensus_population_per_ha", + "path": "", + "format": "PostgreSQL", + "encoding": "UTF-8", + "schema": { + "fields": [ + { + "name": "id", + "description": "Unique identifier", + "type": "integer", + "unit": "none" + }, + { + "name": "grid_id", + "description": "Grid number of source", + "type": "string", + "unit": "none" + }, + { + "name": "population", + "description": "Number of registred residents", + "type": "integer", + "unit": "resident" + }, + { + "name": "geom_point", + "description": "Geometry centroid of zensus cell (CRS: ERTS89/EPSG:3035)", + "type": "Geometry", + "unit": "none" + }, + { + "name": "geom", + "description": "Geometry of zensus cell (CRS: ERTS89/EPSG:3035)", + "type": "Geometry", + "unit": "none" + } + ], + "primaryKey": "id" + }, + "dialect": { + "delimiter": "", + "decimalSeparator": "." + } + } + ], + "review": { + "path": "", + "badge": "" + }, + "metaMetadata": { + "metadataVersion": "OEP-1.4.1", + "metadataLicense": { + "name": "CC0-1.0", + "title": "Creative Commons Zero v1.0 Universal", + "path": "https://creativecommons.org/publicdomain/zero/1.0/" + } + }, + "_comment": { + "metadata": "Metadata documentation and explanation (https://github.com/OpenEnergyPlatform/oemetadata/blob/master/metadata/v141/metadata_key_description.md)", + "dates": "Dates and time must follow the ISO8601 including time zone (YYYY-MM-DD or YYYY-MM-DDThh:mm:ss\u00b1hh)", + "units": "Use a space between numbers and units (100 m)", + "languages": "Languages must follow the IETF (BCP47) format (en-GB, en-US, de-DE)", + "licenses": "License name must follow the SPDX License List (https://spdx.org/licenses/)", + "review": "Following the OEP Data Review (https://github.com/OpenEnergyPlatform/data-preprocessing/wiki)", + "none": "If not applicable use (none)" + } +} diff --git a/src/egon/data/metadata/society.egon_destatis_zensus_apartment_building_population_per_ha.json b/src/egon/data/metadata/society.egon_destatis_zensus_apartment_building_population_per_ha.json new file mode 100644 index 000000000..a48513071 --- /dev/null +++ b/src/egon/data/metadata/society.egon_destatis_zensus_apartment_building_population_per_ha.json @@ -0,0 +1,154 @@ +{ + "name": "society.egon_destatis_zensus_apartment_building_population_per_ha", + "title": "eGon - Zensus 2011 - Apartments, Buildings, Population per hectar", + "id": "WILL_BE_SET_AT_PUBLICATION", + "description": "Apartments, Buildings, Population from national census in Germany in 2011.", + "language": "undefined", + "keywords": [], + "publicationDate": "2023-03-13", + "context": { + "homepage": "https://ego-n.org/", + "documentation": "https://egon-data.readthedocs.io/en/latest/", + "sourceCode": "https://github.com/openego/eGon-data", + "contact": "https://ego-n.org/partners/", + "grantNo": "03EI1002", + "fundingAgency": "Bundesministerium f\\u00fcr Wirtschaft und Energie", + "fundingAgencyLogo": "https://www.innovation-beratung-foerderung.de/INNO/Redaktion/DE/Bilder/Titelbilder/titel_foerderlogo_bmwi.jpg?__blob=normal&v=3", + "publisherLogo": "https://ego-n.org/images/eGon_logo_noborder_transbg.svg" + }, + "spatial": { + "location": "", + "extent": "Germany", + "resolution": "1 ha" + }, + "temporal": {}, + "sources": [ + { + "title": "Statistisches Bundesamt (Destatis) - Ergebnisse des Zensus 2011 zum Download", + "description": "Als Download bieten wir Ihnen auf dieser Seite zus\\u00e4tzlich zur Zensusdatenbank CSV- und teilweise Excel-Tabellen mit umfassenden Personen-, Haushalts- und Familien- sowie Geb\\u00e4ude- und Wohnungsmerkmalen. Die Ergebnisse liegen auf Bundes-, L\\u00e4nder-, Kreis- und Gemeindeebene vor. Au\\u00dferdem sind einzelne Ergebnisse f\\u00fcr Gitterzellen verf\\u00fcgbar.", + "path": "https://www.zensus2011.de/DE/Home/Aktuelles/DemografischeGrunddaten.html", + "licenses": [ + { + "name": "dl-by-de/2.0", + "title": "Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0", + "path": "www.govdata.de/dl-de/by-2-0", + "instruction": "Jede Nutzung ist unter den Bedingungen dieser \\u201eDatenlizenz Deutschland - Namensnennung - Version 2.0 zul\\u00e4ssig.\\nDie bereitgestellten Daten und Metadaten d\\u00fcrfen f\\u00fcr die kommerzielle und nicht kommerzielle Nutzung insbesondere:(1) vervielf\\u00e4ltigt, ausgedruckt, pr\\u00e4sentiert, ver\\u00e4ndert, bearbeitet sowie an Dritte \\u00fcbermittelt werden;\\n (2) mit eigenen Daten und Daten Anderer zusammengef\\u00fchrt und zu selbst\\u00e4ndigen neuen Datens\\u00e4tzen verbunden werden;\\n (3) in interne und externe Gesch\\u00e4ftsprozesse, Produkte und Anwendungen in \\u00f6ffentlichen und nicht \\u00f6ffentlichen elektronischen Netzwerken eingebunden werden.\\nBei der Nutzung ist sicherzustellen, dass folgende Angaben als Quellenvermerk enthalten sind:\\n(1) Bezeichnung des Bereitstellers nach dessen Ma\\u00dfgabe,\\n(2) der Vermerk Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 sowie\\n(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.\\nVer\\u00e4nderungen, Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk mit dem Hinweis zu versehen, dass die Daten ge\\u00e4ndert wurden.", + "attribution": "\\u00a9 Statistische \\u00c4mter des Bundes und der L\\u00e4nder 2014" + } + ] + }, + { + "title": "Dokumentation - Zensus 2011 - Methoden und Verfahren", + "description": "Diese Publikation beschreibt ausf\\u00fchrlich die Methoden und Verfahren des registergest\\u00fctzten Zensus 2011; von der Datengewinnung und -aufbereitung bis hin zur Ergebniserstellung und Geheimhaltung. Der vorliegende Band wurde von den Statistischen \\u00c4mtern des Bundes und der L\\u00e4nder im Juni 2015 ver\\u00f6ffentlicht.", + "path": "https://www.destatis.de/DE/Publikationen/Thematisch/Bevoelkerung/Zensus/ZensusBuLaMethodenVerfahren5121105119004.pdf?__blob=publicationFile", + "licenses": [ + { + "title": "Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0", + "path": "www.govdata.de/dl-de/by-2-0", + "instruction": "Jede Nutzung ist unter den Bedingungen dieser \\u201eDatenlizenz Deutschland - Namensnennung - Version 2.0 zul\\u00e4ssig.\\nDie bereitgestellten Daten und Metadaten d\\u00fcrfen f\\u00fcr die kommerzielle und nicht kommerzielle Nutzung insbesondere:(1) vervielf\\u00e4ltigt, ausgedruckt, pr\\u00e4sentiert, ver\\u00e4ndert, bearbeitet sowie an Dritte \\u00fcbermittelt werden;\\n (2) mit eigenen Daten und Daten Anderer zusammengef\\u00fchrt und zu selbst\\u00e4ndigen neuen Datens\\u00e4tzen verbunden werden;\\n (3) in interne und externe Gesch\\u00e4ftsprozesse, Produkte und Anwendungen in \\u00f6ffentlichen und nicht \\u00f6ffentlichen elektronischen Netzwerken eingebunden werden.\\nBei der Nutzung ist sicherzustellen, dass folgende Angaben als Quellenvermerk enthalten sind:\\n(1) Bezeichnung des Bereitstellers nach dessen Ma\\u00dfgabe,\\n(2) der Vermerk Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 sowie\\n(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.\\nVer\\u00e4nderungen, Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk mit dem Hinweis zu versehen, dass die Daten ge\\u00e4ndert wurden.", + "attribution": "\\u00a9 Statistisches Bundesamt, Wiesbaden 2015 (im Auftrag der Herausgebergemeinschaft)", + "name": "dl-by-de/2.0" + } + ] + } + ], + "licenses": [ + { + "name": "dl-by-de/2.0", + "title": "Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0", + "path": "www.govdata.de/dl-de/by-2-0", + "instruction": "Jede Nutzung ist unter den Bedingungen dieser \\u201eDatenlizenz Deutschland - Namensnennung - Version 2.0 zul\\u00e4ssig.\\nDie bereitgestellten Daten und Metadaten d\\u00fcrfen f\\u00fcr die kommerzielle und nicht kommerzielle Nutzung insbesondere:(1) vervielf\\u00e4ltigt, ausgedruckt, pr\\u00e4sentiert, ver\\u00e4ndert, bearbeitet sowie an Dritte \\u00fcbermittelt werden;\\n (2) mit eigenen Daten und Daten Anderer zusammengef\\u00fchrt und zu selbst\\u00e4ndigen neuen Datens\\u00e4tzen verbunden werden;\\n (3) in interne und externe Gesch\\u00e4ftsprozesse, Produkte und Anwendungen in \\u00f6ffentlichen und nicht \\u00f6ffentlichen elektronischen Netzwerken eingebunden werden.\\nBei der Nutzung ist sicherzustellen, dass folgende Angaben als Quellenvermerk enthalten sind:\\n(1) Bezeichnung des Bereitstellers nach dessen Ma\\u00dfgabe,\\n(2) der Vermerk Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 sowie\\n(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.\\nVer\\u00e4nderungen, Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk mit dem Hinweis zu versehen, dass die Daten ge\\u00e4ndert wurden.", + "attribution": "\\u00a9 Statistische \\u00c4mter des Bundes und der L\\u00e4nder 2014; \\u00a9 Statistisches Bundesamt, Wiesbaden 2015 (Daten ver\\u00e4ndert)" + } + ], + "contributors": [ + { + "title": "nailend", + "email": "https://github.com/nailend", + "date": "2023-03-13", + "object": "metadata", + "comment": "Create metadata" + } + ], + "resources": [ + { + "profile": "tabular-data-resource", + "name": "society.egon_destatis_zensus_apartment_building_population_per_ha", + "path": "", + "format": "PostgreSQL", + "encoding": "UTF-8", + "schema": { + "fields": [ + { + "name": "grid_id", + "description": "Grid number of source", + "type": "string", + "unit": "none" + }, + { + "name": "zensus_population_id", + "description": "Unique identifier", + "type": "integer", + "unit": "none" + }, + { + "name": "building_count", + "description": "Number of buildings within zensus cell ", + "type": "smallint", + "unit": "none" + }, + { + "name": "apartment_count", + "description": "Number of appartments within zensus cells", + "type": "smallint", + "unit": "none" + }, + { + "name": "population", + "description": "Number of registred residents", + "type": "integer", + "unit": "resident" + }, + { + "name": "geom", + "description": "Geometry of zensus cell (CRS: ERTS89/EPSG:3035)", + "type": "Geometry", + "unit": "none" + }, + { + "name": "geom_point", + "description": "Geometry centroid of zensus cell (CRS: ERTS89/EPSG:3035)", + "type": "Geometry", + "unit": "none" + } + ], + "primaryKey": "none" + }, + "dialect": { + "delimiter": "", + "decimalSeparator": "." + } + } + ], + "metaMetadata": { + "metadataVersion": "OEP-1.4.1", + "metadataLicense": { + "name": "CC0-1.0", + "title": "Creative Commons Zero v1.0 Universal", + "path": "https://creativecommons.org/publicdomain/zero/1.0/" + } + }, + "review": { + "path": "", + "badge": "" + }, + "_comment": { + "metadata": "Metadata documentation and explanation (https://github.com/OpenEnergyPlatform/oemetadata/blob/master/metadata/v141/metadata_key_description.md)", + "dates": "Dates and time must follow the ISO8601 including time zone (YYYY-MM-DD or YYYY-MM-DDThh:mm:ss\u00b1hh)", + "units": "Use a space between numbers and units (100 m)", + "languages": "Languages must follow the IETF (BCP47) format (en-GB, en-US, de-DE)", + "licenses": "License name must follow the SPDX License List (https://spdx.org/licenses/)", + "review": "Following the OEP Data Review (https://github.com/OpenEnergyPlatform/data-preprocessing/wiki)", + "none": "If not applicable use (none)" + } +}