diff --git a/ding0/io/ego_dp_versioning.py b/ding0/io/ego_dp_versioning.py new file mode 100644 index 00000000..8a90131e --- /dev/null +++ b/ding0/io/ego_dp_versioning.py @@ -0,0 +1,155 @@ +# """This file is part of DINGO, the DIstribution Network GeneratOr. +# DINGO is a tool to generate synthetic medium and low voltage power +# distribution grids based on open data. +# +# It is developed in the project open_eGo: https://openegoproject.wordpress.com +# +# DING0 lives at github: https://github.com/openego/ding0/ +# The documentation is available on RTD: http://ding0.readthedocs.io""" +# +# __copyright__ = "Reiner Lemoine Institut gGmbH" +# __license__ = "GNU Affero General Public License Version 3 (AGPL-3.0)" +# __url__ = "https://github.com/openego/ding0/blob/master/LICENSE" +# __author__ = "jh-RLI" +# +# +# import json +# import os +# +# from egoio.tools.db import connection +# +# from sqlalchemy import MetaData +# +# # set your Table names +# DING0_TABLES = {'versioning': 'ego_ding0_versioning', +# 'line': 'ego_ding0_line', +# 'lv_branchtee': 'ego_ding0_lv_branchtee', +# 'lv_generator': 'ego_ding0_lv_generator', +# 'lv_load': 'ego_ding0_lv_load', +# 'lv_grid': 'ego_ding0_lv_grid', +# 'lv_station': 'ego_ding0_lv_station', +# 'mvlv_transformer': 'ego_ding0_mvlv_transformer', +# 'mvlv_mapping': 'ego_ding0_mvlv_mapping', +# 'mv_branchtee': 'ego_ding0_mv_branchtee', +# 'mv_circuitbreaker': 'ego_ding0_mv_circuitbreaker', +# 'mv_generator': 'ego_ding0_mv_generator', +# 'mv_load': 'ego_ding0_mv_load', +# 'mv_grid': 'ego_ding0_mv_grid', +# 'mv_station': 'ego_ding0_mv_station', +# 'hvmv_transformer': 'ego_ding0_hvmv_transformer'} +# +# # #########SQLAlchemy and DB table################ +# #source +# oedb_engine = connection(section='oedb') +# # Testing Database -> destination +# reiners_engine = connection(section='reiners_db') +# +# REFLICTED_SCHEMA = "model_draft" +# VERSIONING_SCHEMA = "grid" +# +# META = MetaData() +# META.reflect(bind=reiners_engine, schema=REFLICTED_SCHEMA, only=DING0_TABLES['versioning', 'line', 'lv_branchtee', +# 'lv_generator', 'lv_load', 'lv_grid', +# 'lv_station', 'mvlv_transformer', +# 'mvlv_mapping', 'mv_branchtee', +# 'mv_circuitbreaker', 'mv_generator', +# 'mv_load', 'mv_grid', 'mv_station', +# 'hvmv_transformer']) +# # ################################################ +# +# tables = META.metadata.tables +# for tbl in tables: +# print ('##################################') +# print (tbl) +# print ( tables[tbl].select()) +# data = oedb_engine.execute(tables[tbl].select()).fetchall() +# for a in data: print(a) +# if data: +# print (tables[tbl].insert()) +# reiners_engine.execute( tables[tbl].insert(), data) + +#!/usr/bin/env python + +import sys +from sqlalchemy import create_engine, MetaData, Table +from sqlalchemy.orm import sessionmaker +from sqlalchemy.ext.declarative import declarative_base + +from egoio.tools.db import connection + + +# set your Table names +# DING0_TABLES = {'versioning': 'ego_ding0_versioning', +# 'line': 'ego_ding0_line', +# 'lv_branchtee': 'ego_ding0_lv_branchtee', +# 'lv_generator': 'ego_ding0_lv_generator', +# 'lv_load': 'ego_ding0_lv_load', +# 'lv_grid': 'ego_ding0_lv_grid', +# 'lv_station': 'ego_ding0_lv_station', +# 'mvlv_transformer': 'ego_ding0_mvlv_transformer', +# 'mvlv_mapping': 'ego_ding0_mvlv_mapping', +# 'mv_branchtee': 'ego_ding0_mv_branchtee', +# 'mv_circuitbreaker': 'ego_ding0_mv_circuitbreaker', +# 'mv_generator': 'ego_ding0_mv_generator', +# 'mv_load': 'ego_ding0_mv_load', +# 'mv_grid': 'ego_ding0_mv_grid', +# 'mv_station': 'ego_ding0_mv_station', +# 'hvmv_transformer': 'ego_ding0_hvmv_transformer'} + +DING0_TABLES = {'mv_generator': 'ego_ding0_mv_generator'} + + +def get_table_names(t): + tables = [] + for k, v in t.items(): + tables.append(v) + return tables + + +def make_session(engine): + Session = sessionmaker(bind=engine) + return Session(), engine + + +def pull_data(from_db, s_schema, to_db, d_schema, tables): + source, sengine = make_session(from_db) + smeta = MetaData(bind=sengine, schema=s_schema) + destination, dengine = make_session(to_db) + + for table_name in get_table_names(DING0_TABLES): + print('Processing', table_name) + print('Pulling schema from source server') + table = Table(table_name, smeta, autoload=True) + table.schema = d_schema + print('Creating table on destination server') + table.metadata.create_all(dengine, checkfirst=True) + new_record = quick_mapper(table) + columns = table.columns.keys() + print('Transferring records') + for record in source.query(table).all(): + data = dict( + [(str(column), getattr(record, column)) for column in columns] + ) + destination.merge(new_record(**data)) + print('Committing changes') + destination.commit() + + +def quick_mapper(table): + Base = declarative_base() + class GenericMapper(Base): + __table__ = table + return GenericMapper + + +if __name__ == '__main__': + # source + oedb_engine = connection(section='oedb') + # # Testing Database -> destination + reiners_engine = connection(section='reiners_db') + + SOURCE_SCHEMA = 'model_draft' + DESTINATION_SCHEMA = 'grid' + tables = get_table_names(DING0_TABLES) + + pull_data(oedb_engine, SOURCE_SCHEMA, oedb_engine, DESTINATION_SCHEMA, tables) diff --git a/ding0/io/metadatastrings/hvmv_transformer.json b/ding0/io/metadatastrings/hvmv_transformer.json index a61c1ecc..e44eac67 100644 --- a/ding0/io/metadatastrings/hvmv_transformer.json +++ b/ding0/io/metadatastrings/hvmv_transformer.json @@ -25,7 +25,7 @@ {"name": "Jonas Huber", "email": "jonas.huber@rl-institut.de", "date": "2018-08-17", "comment": "Create metadata"}, {"name": "Jonas Huber", "email": "jonas.huber@rl-institut.de", "date": "2018-09-12", "comment": "Update metadata section source"} ], "resources": [ - {"name": "model_draft.ego_grid_ding0_hvmv_transformer", + {"name": "grid.ego_grid_ding0_hvmv_transformer", "format": "PostgreSQL", "fields": [ {"name": "id","description": "unambiguous unique numer","unit": "integer"}, diff --git a/ding0/io/metadatastrings/line.json b/ding0/io/metadatastrings/line.json index a577a50c..1f2c8449 100644 --- a/ding0/io/metadatastrings/line.json +++ b/ding0/io/metadatastrings/line.json @@ -25,7 +25,7 @@ {"name": "Jonas Huber", "email": "jonas.huber@rl-institut.de", "date": "2018-08-17", "comment": "Create metadata"}, {"name": "Jonas Huber", "email": "jonas.huber@rl-institut.de", "date": "2018-09-12", "comment": "Update metadata section source"} ], "resources": [ - {"name": "model_draft.ego_grid_ding0_line", + {"name": "grid.ego_grid_ding0_line", "format": "PostgreSQL", "fields": [ {"name": "id","description": "unambiguous unique numer","unit": "integer"}, diff --git a/ding0/io/metadatastrings/lv_branchtee.json b/ding0/io/metadatastrings/lv_branchtee.json index 4d9a17eb..3b29c6ca 100644 --- a/ding0/io/metadatastrings/lv_branchtee.json +++ b/ding0/io/metadatastrings/lv_branchtee.json @@ -25,7 +25,7 @@ {"name": "Jonas Huber", "email": "jonas.huber@rl-institut.de", "date": "2018-08-17", "comment": "Create metadata"}, {"name": "Jonas Huber", "email": "jonas.huber@rl-institut.de", "date": "2018-09-12", "comment": "Update metadata section source"} ], "resources": [ - {"name": "model_draft.ego_grid_ding0_lv_branchtee", + {"name": "grid.ego_grid_ding0_lv_branchtee", "format": "PostgreSQL", "fields": [ {"name": "id","discription": "unambiguous unique numer","unit": "integer"}, diff --git a/ding0/io/metadatastrings/lv_generator.json b/ding0/io/metadatastrings/lv_generator.json index 68ef3661..769164e2 100644 --- a/ding0/io/metadatastrings/lv_generator.json +++ b/ding0/io/metadatastrings/lv_generator.json @@ -25,7 +25,7 @@ {"name": "Jonas Huber", "email": "jonas.huber@rl-institut.de", "date": "2018-08-17", "comment": "Create metadata"}, {"name": "Jonas Huber", "email": "jonas.huber@rl-institut.de", "date": "2018-09-12", "comment": "Update metadata section source"} ], "resources": [ - {"name": "model_draft.ego_grid_ding0_lv_generator", + {"name": "grid.ego_grid_ding0_lv_generator", "format": "PostgreSQL", "fields": [ {"name": "id","description": "unambiguous unique numer","unit": "integer"}, diff --git a/ding0/io/metadatastrings/lv_grid.json b/ding0/io/metadatastrings/lv_grid.json index 4ef26f8c..263d8a1d 100644 --- a/ding0/io/metadatastrings/lv_grid.json +++ b/ding0/io/metadatastrings/lv_grid.json @@ -25,7 +25,7 @@ {"name": "Jonas Huber", "email": "jonas.huber@rl-institut.de", "date": "2018-08-17", "comment": "Create metadata"}, {"name": "Jonas Huber", "email": "jonas.huber@rl-institut.de", "date": "2018-09-12", "comment": "Update metadata section source"} ], "resources": [ - {"name": "model_draft.ego_grid_ding0_lv_grid", "format": "PostgreSQL", "fields": [ + {"name": "grid.ego_grid_ding0_lv_grid", "format": "PostgreSQL", "fields": [ {"name": "id","description": "unambiguous unique numer","unit": "integer"}, {"name": "run_id","description": "time and date of table generation","unit": "yyyyMMddhhmmss"}, {"name": "id_db","description": "unambiguous number of LV-Grid","unit": "integer"}, diff --git a/ding0/io/metadatastrings/lv_load.json b/ding0/io/metadatastrings/lv_load.json index 66f3f992..7276db63 100644 --- a/ding0/io/metadatastrings/lv_load.json +++ b/ding0/io/metadatastrings/lv_load.json @@ -25,7 +25,7 @@ {"name": "Jonas Huber", "email": "jonas.huber@rl-institut.de", "date": "2018-08-17", "comment": "Create metadata"}, {"name": "Jonas Huber", "email": "jonas.huber@rl-institut.de", "date": "2018-09-12", "comment": "Update metadata section source"} ], "resources": [ - {"name": "model_draft.ego_grid_ding0_lv_load", + {"name": "grid.ego_grid_ding0_lv_load", "format": "PostgreSQL", "fields": [ {"name": "id","description": "unambiguous unique numer","unit": "integer"}, diff --git a/ding0/io/metadatastrings/lv_station.json b/ding0/io/metadatastrings/lv_station.json index 6f83d319..b57b2bc3 100644 --- a/ding0/io/metadatastrings/lv_station.json +++ b/ding0/io/metadatastrings/lv_station.json @@ -25,7 +25,7 @@ {"name": "Jonas Huber", "email": "jonas.huber@rl-institut.de", "date": "2018-08-17", "comment": "Create metadata"}, {"name": "Jonas Huber", "email": "jonas.huber@rl-institut.de", "date": "2018-09-12", "comment": "Update metadata section source"} ], "resources": [ - {"name": "model_draft.ego_grid_ding0_lv_station", + {"name": "grid.ego_grid_ding0_lv_station", "format": "PostgreSQL", "fields": [ {"name": "id","description": "unambiguous unique numer","unit": "integer"}, diff --git a/ding0/io/metadatastrings/mv_branchtee.json b/ding0/io/metadatastrings/mv_branchtee.json index 495b0f04..7663db04 100644 --- a/ding0/io/metadatastrings/mv_branchtee.json +++ b/ding0/io/metadatastrings/mv_branchtee.json @@ -25,7 +25,7 @@ {"name": "Jonas Huber", "email": "jonas.huber@rl-institut.de", "date": "2018-08-17", "comment": "Create metadata"}, {"name": "Jonas Huber", "email": "jonas.huber@rl-institut.de", "date": "2018-09-12", "comment": "Update metadata section source"} ], "resources": [ - {"name": "model_draft.ego_grid_ding0_mv_branchtee", + {"name": "grid.ego_grid_ding0_mv_branchtee", "format": "PostgreSQL", "fields": [ {"name": "id","description": "unambiguous unique numer","unit": "integer"}, diff --git a/ding0/io/metadatastrings/mv_circuitbreaker.json b/ding0/io/metadatastrings/mv_circuitbreaker.json index 2cd2e00d..81b45a77 100644 --- a/ding0/io/metadatastrings/mv_circuitbreaker.json +++ b/ding0/io/metadatastrings/mv_circuitbreaker.json @@ -25,7 +25,7 @@ {"name": "Jonas Huber", "email": "jonas.huber@rl-institut.de", "date": "2018-08-17", "comment": "Create metadata"}, {"name": "Jonas Huber", "email": "jonas.huber@rl-institut.de", "date": "2018-09-12", "comment": "Update metadata section source"} ], "resources": [ - {"name": "model_draft.ego_grid_ding0_mv_circuitbreaker", + {"name": "grid.ego_grid_ding0_mv_circuitbreaker", "format": "PostgreSQL", "fields": [ {"name": "id","description": "unambiguous unique numer","unit": "integer"}, diff --git a/ding0/io/metadatastrings/mv_generator.json b/ding0/io/metadatastrings/mv_generator.json index 4aa15423..a13ce158 100644 --- a/ding0/io/metadatastrings/mv_generator.json +++ b/ding0/io/metadatastrings/mv_generator.json @@ -25,7 +25,7 @@ {"name": "Jonas Huber", "email": "jonas.huber@rl-institut.de", "date": "2018-08-17", "comment": "Create metadata"}, {"name": "Jonas Huber", "email": "jonas.huber@rl-institut.de", "date": "2018-09-12", "comment": "Update metadata section source"} ], "resources": [ - {"name": "model_draft.ego_grid_ding0_mv_generator", + {"name": "grid.ego_grid_ding0_mv_generator", "format": "PostgreSQL", "fields": [ {"name": "id","description": "unambiguous unique numer","unit": "integer"}, diff --git a/ding0/io/metadatastrings/mv_grid.json b/ding0/io/metadatastrings/mv_grid.json index 55dddaf0..e7632581 100644 --- a/ding0/io/metadatastrings/mv_grid.json +++ b/ding0/io/metadatastrings/mv_grid.json @@ -25,7 +25,7 @@ {"name": "Jonas Huber", "email": "jonas.huber@rl-institut.de", "date": "2018-08-17", "comment": "Create metadata"}, {"name": "Jonas Huber", "email": "jonas.huber@rl-institut.de", "date": "2018-09-12", "comment": "Update metadata section source"} ], "resources": [ - {"name": "model_draft.ego_grid_ding0_mv_grid", + {"name": "grid.ego_grid_ding0_mv_grid", "format": "PostgreSQL", "fields": [ {"name": "id","description": "unambiguous unique numer","unit": "integer"}, diff --git a/ding0/io/metadatastrings/mv_load.json b/ding0/io/metadatastrings/mv_load.json index 64856203..5b246425 100644 --- a/ding0/io/metadatastrings/mv_load.json +++ b/ding0/io/metadatastrings/mv_load.json @@ -25,7 +25,7 @@ {"name": "Jonas Huber", "email": "jonas.huber@rl-institut.de", "date": "2018-08-17", "comment": "Create metadata"}, {"name": "Jonas Huber", "email": "jonas.huber@rl-institut.de", "date": "2018-09-12", "comment": "Update metadata section source"} ], "resources": [ - {"name": "model_draft.ego_grid_ding0_mv_load", + {"name": "grid.ego_grid_ding0_mv_load", "format": "PostgreSQL", "fields": [ {"name": "id","description": "unambiguous unique numer","unit": "integer"}, diff --git a/ding0/io/metadatastrings/mv_station.json b/ding0/io/metadatastrings/mv_station.json index 5799d419..a28d6a3f 100644 --- a/ding0/io/metadatastrings/mv_station.json +++ b/ding0/io/metadatastrings/mv_station.json @@ -25,7 +25,7 @@ {"name": "Jonas Huber", "email": "jonas.huber@rl-institut.de", "date": "2018-08-17", "comment": "Create metadata"}, {"name": "Jonas Huber", "email": "jonas.huber@rl-institut.de", "date": "2018-09-12", "comment": "Update metadata section source"} ], "resources": [ - {"name": "model_draft.ego_grid_ding0_mv_station", + {"name": "grid.ego_grid_ding0_mv_station", "format": "PostgreSQL", "fields": [ {"name": "id","description": "unambiguous unique numer","unit": "integer"}, diff --git a/ding0/io/metadatastrings/mvlv_mapping.json b/ding0/io/metadatastrings/mvlv_mapping.json index 48ce5032..831f4bb2 100644 --- a/ding0/io/metadatastrings/mvlv_mapping.json +++ b/ding0/io/metadatastrings/mvlv_mapping.json @@ -25,7 +25,7 @@ {"name": "Jonas Huber", "email": "jonas.huber@rl-institut.de", "date": "2018-08-17", "comment": "Create metadata"}, {"name": "Jonas Huber", "email": "jonas.huber@rl-institut.de", "date": "2018-09-12", "comment": "Update metadata section source"} ], "resources": [ - {"name": "model_draft.ego_grid_ding0_mvlv_mapping", + {"name": "grid.ego_grid_ding0_mvlv_mapping", "format": "PostgreSQL", "fields": [ {"name": "id","description": "unambiguous unique numer","unit": "integer"}, diff --git a/ding0/io/metadatastrings/mvlv_transformer.json b/ding0/io/metadatastrings/mvlv_transformer.json index 290982c9..ff09a4fb 100644 --- a/ding0/io/metadatastrings/mvlv_transformer.json +++ b/ding0/io/metadatastrings/mvlv_transformer.json @@ -25,7 +25,7 @@ {"name": "Jonas Huber", "email": "jonas.huber@rl-institut.de", "date": "2018-08-17", "comment": "Create metadata"}, {"name": "Jonas Huber", "email": "jonas.huber@rl-institut.de", "date": "2018-09-12", "comment": "Update metadata section source"} ], "resources": [ - {"name": "model_draft.ego_grid_ding0_mvlv_transformer", + {"name": "grid.ego_grid_ding0_mvlv_transformer", "format": "PostgreSQL", "fields": [ {"name": "id","description": "unambiguous unique numer","unit": "integer"}, diff --git a/ding0/io/metadatastrings/versioning.json b/ding0/io/metadatastrings/versioning.json index 51dec620..f67b5bcc 100644 --- a/ding0/io/metadatastrings/versioning.json +++ b/ding0/io/metadatastrings/versioning.json @@ -25,7 +25,7 @@ {"name": "Jonas Huber", "email": "jonas.huber@rl-institut.de", "date": "2018-08-17", "comment": "Create metadata"}, {"name": "Jonas Huber", "email": "jonas.huber@rl-institut.de", "date": "2018-09-12", "comment": "Update metadata section source"} ], "resources": [ - {"name": "model_draft.ego_grid_ding0_versioning", + {"name": "grid.ego_grid_ding0_versioning", "format": "PostgreSQL", "fields": [ {"name": "id","description": "unambiguous unique numer","unit": "integer"},