From ee9b75f70b7107c4d9f97f6526177cf74bbbbd13 Mon Sep 17 00:00:00 2001 From: ulf Date: Tue, 30 May 2017 18:26:12 +0200 Subject: [PATCH 01/65] first draft for extra functionality ll minimization --- etrago/extra_high_voltage_SH_appl.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/etrago/extra_high_voltage_SH_appl.py b/etrago/extra_high_voltage_SH_appl.py index 987f0286..5666794a 100644 --- a/etrago/extra_high_voltage_SH_appl.py +++ b/etrago/extra_high_voltage_SH_appl.py @@ -97,7 +97,7 @@ #add connection from Luebeck to Siems network.add("Bus", "Siems220",carrier='AC', v_nom=220, x=10.760835, y=53.909745) -network.add("Transformer", "Siems220_380", bus0="25536", bus1="Siems220", x=1.29960, tap_ratio=1) +network.add("Transformer", "Siems220_380", bus0="25536", bus1="Siems220", x=1.29960, tap_ratio=1, s_nom=1600) network.add("Line","LuebeckSiems", bus0="26387",bus1="Siems220", x=0.0001, s_nom=1600) @@ -115,6 +115,16 @@ #load shedding in order to hunt infeasibilities #load_shedding(network) + + +#def extra_functionality(network,snapshots): + +# def line_loading(network,snapshot): +# return sum over all passive branch flows *2 + +# #add an additional objective function in order to minimize line loading +# network.model.passive_branch_p = Objective(rule=line_loading, sense=minimize) + # start powerflow calculations network.lopf(snapshots, solver_name='gurobi') From c695b8f322b624924adb85fd9913f2d9b8d1992b Mon Sep 17 00:00:00 2001 From: ulf Date: Wed, 31 May 2017 16:19:24 +0200 Subject: [PATCH 02/65] try out extr_functionality --- etrago/extra_high_voltage_SH_appl.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/etrago/extra_high_voltage_SH_appl.py b/etrago/extra_high_voltage_SH_appl.py index 5666794a..04cd4e36 100644 --- a/etrago/extra_high_voltage_SH_appl.py +++ b/etrago/extra_high_voltage_SH_appl.py @@ -21,7 +21,7 @@ EgoGridPfHvLoadPqSet as LoadPqSet, EgoGridPfHvSource as Source from cluster.networkclustering import busmap_from_psql, cluster_on_extra_high_voltage from extras.utilities import load_shedding - +from pyomo.environ import Objective session = oedb_session() @@ -108,25 +108,26 @@ network.generators.control="PV" -busmap = busmap_from_psql(network, session, scn_name=scenario) +#busmap = busmap_from_psql(network, session, scn_name=scenario) -network = cluster_on_extra_high_voltage(network, busmap, with_time=True) +#network = cluster_on_extra_high_voltage(network, busmap, with_time=True) #load shedding in order to hunt infeasibilities #load_shedding(network) -#def extra_functionality(network,snapshots): - -# def line_loading(network,snapshot): -# return sum over all passive branch flows *2 - -# #add an additional objective function in order to minimize line loading -# network.model.passive_branch_p = Objective(rule=line_loading, sense=minimize) +def extra_functionality(network,snapshots): + network.model.objective.add(sum(network.model.passive_branch_p)) +# def line_loading(model): +# +# return sum(model.passive_branch_p) +# +# #add an additional objective function in order to minimize line loading +# network.model.passive_branch_p = Objective(rule=line_loading) # start powerflow calculations -network.lopf(snapshots, solver_name='gurobi') +network.lopf(snapshots, solver_name='gurobi', extra_functionality=extra_functionality) network.model.write('/home/ulf/file.lp', io_options={'symbolic_solver_labels':True}) From eb84b9afb857ec6fe1953dde351b2cce100ba0ba Mon Sep 17 00:00:00 2001 From: ulf Date: Wed, 31 May 2017 17:38:15 +0200 Subject: [PATCH 03/65] no comment --- etrago/extra_high_voltage_SH_appl.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etrago/extra_high_voltage_SH_appl.py b/etrago/extra_high_voltage_SH_appl.py index 04cd4e36..c6f46d69 100644 --- a/etrago/extra_high_voltage_SH_appl.py +++ b/etrago/extra_high_voltage_SH_appl.py @@ -118,7 +118,7 @@ def extra_functionality(network,snapshots): - network.model.objective.add(sum(network.model.passive_branch_p)) + network.model.objective = network.model.objective.add(expr=sum(network.lines_t.p0.abs().sum()), index= None) # def line_loading(model): # # return sum(model.passive_branch_p) @@ -127,7 +127,7 @@ def extra_functionality(network,snapshots): # network.model.passive_branch_p = Objective(rule=line_loading) # start powerflow calculations -network.lopf(snapshots, solver_name='gurobi', extra_functionality=extra_functionality) +network.lopf(snapshots, solver_name='gurobi', extra_functionality=None) network.model.write('/home/ulf/file.lp', io_options={'symbolic_solver_labels':True}) From cb1a018d0acdf28cd0e59bfd609628f6c411502d Mon Sep 17 00:00:00 2001 From: ulf Date: Thu, 1 Jun 2017 14:26:37 +0200 Subject: [PATCH 04/65] just trying --- etrago/extra_high_voltage_SH_appl.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/etrago/extra_high_voltage_SH_appl.py b/etrago/extra_high_voltage_SH_appl.py index c6f46d69..7987cb1e 100644 --- a/etrago/extra_high_voltage_SH_appl.py +++ b/etrago/extra_high_voltage_SH_appl.py @@ -21,7 +21,7 @@ EgoGridPfHvLoadPqSet as LoadPqSet, EgoGridPfHvSource as Source from cluster.networkclustering import busmap_from_psql, cluster_on_extra_high_voltage from extras.utilities import load_shedding -from pyomo.environ import Objective +from pyomo.environ import Objective, Var session = oedb_session() @@ -115,19 +115,25 @@ #load shedding in order to hunt infeasibilities #load_shedding(network) +# Variables +#model.X = Var( model .P) + def extra_functionality(network,snapshots): - network.model.objective = network.model.objective.add(expr=sum(network.lines_t.p0.abs().sum()), index= None) -# def line_loading(model): + + + network.model.objective = network.model.objective.add(expr=network.model.passive_branch_p['Line'], index=None) +# def line_loading(model,snapshot): # -# return sum(model.passive_branch_p) -# +# return model.passive_branch_p['line',snapshot] + # #add an additional objective function in order to minimize line loading -# network.model.passive_branch_p = Objective(rule=line_loading) +# network.model.objective.add(expr=line_loading, index=None) +# #network.model.line_loading = Objective(list(snapshots),rule=line_loading) # start powerflow calculations -network.lopf(snapshots, solver_name='gurobi', extra_functionality=None) +network.lopf(snapshots, solver_name='gurobi', extra_functionality=extra_functionality) network.model.write('/home/ulf/file.lp', io_options={'symbolic_solver_labels':True}) From 0bf2723ad3ae04f88f906d47f556f6508464d4c0 Mon Sep 17 00:00:00 2001 From: ulf Date: Thu, 1 Jun 2017 15:44:14 +0200 Subject: [PATCH 05/65] network clustering by k-means successfully tested --- etrago/k_means_testing.py | 146 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 146 insertions(+) create mode 100644 etrago/k_means_testing.py diff --git a/etrago/k_means_testing.py b/etrago/k_means_testing.py new file mode 100644 index 00000000..f7488720 --- /dev/null +++ b/etrago/k_means_testing.py @@ -0,0 +1,146 @@ +"""This is the docstring for the example.py module. Modules names should +have short, all-lowercase names. The module name may have underscores if +this improves readability. +Every module should have a docstring at the very top of the file. The +module's docstring may extend over multiple lines. If your docstring does +extend over multiple lines, the closing three quotation marks must be on +a line by itself, preferably preceded by a blank line.""" + +__copyright__ = "tba" +__license__ = "tba" +__author__ = "tba" + + +from egopowerflow.tools.tools import oedb_session +from egopowerflow.tools.io import get_timerange, import_components, import_pq_sets,\ + add_source_types, create_powerflow_problem +from egopowerflow.tools.plot import add_coordinates, plot_line_loading,\ + plot_stacked_gen +from egoio.db_tables.model_draft import EgoGridPfHvBus as Bus, EgoGridPfHvLine as Line, EgoGridPfHvGenerator as Generator, EgoGridPfHvLoad as Load,\ + EgoGridPfHvTransformer as Transformer, EgoGridPfHvTempResolution as TempResolution, EgoGridPfHvGeneratorPqSet as GeneratorPqSet,\ + EgoGridPfHvLoadPqSet as LoadPqSet, EgoGridPfHvSource as Source +from cluster.networkclustering import busmap_from_psql, cluster_on_extra_high_voltage +from pypsa.networkclustering import busmap_by_kmeans, get_clustering_from_busmap +import pandas as pd +import numpy as np + +session = oedb_session('grids_test') + +scenario = 'SH Status Quo' + +# define relevant tables of generator table +pq_set_cols_1 = ['p_set'] +pq_set_cols_2 = ['q_set'] +p_max_pu = ['p_max_pu'] + +# choose relevant parameters used in pf +temp_id_set = 1 +start_h = 2300 +end_h = 2301 + +# define investigated time range +timerange = get_timerange(session, temp_id_set, TempResolution, start_h, end_h) + +# define relevant tables +tables = [Bus, Line, Generator, Load, Transformer] + +# get components from database tables +components = import_components(tables, session, scenario) + +# create PyPSA powerflow problem +network, snapshots = create_powerflow_problem(timerange, components) + +# import pq-set tables to pypsa network (p_set for generators and loads) +pq_object = [GeneratorPqSet, LoadPqSet] +network = import_pq_sets(session=session, + network=network, + pq_tables=pq_object, + timerange=timerange, + scenario=scenario, + columns=pq_set_cols_1, + start_h=start_h, + end_h=end_h) + +# import pq-set table to pypsa network (q_set for loads) +network = import_pq_sets(session=session, + network=network, + pq_tables=[LoadPqSet], + timerange=timerange, + scenario=scenario, + columns=pq_set_cols_2, + start_h=start_h, + end_h=end_h) + +network = import_pq_sets(session=session, + network=network, + pq_tables=[GeneratorPqSet], + timerange=timerange, + scenario=scenario, + columns=p_max_pu, + start_h=start_h, + end_h=end_h) + +## import time data for storages: +#network = import_pq_sets(session=session, +# network=network, +# pq_tables=[StoragePqSet], +# timerange=timerange, +# scenario=scenario, +# columns=storage_sets, +# start_h=start_h, +# end_h=end_h) + +# add coordinates to network nodes and make ready for map plotting +network = add_coordinates(network) + +# add source names to generators +add_source_types(session, network, table=Source) + +#add connection from Luebeck to Siems +network.add("Bus", "Siems220",carrier='AC', v_nom=220, x=10.760835, y=53.909745) +network.add("Transformer", "Siems220_380", bus0="25536", bus1="Siems220", x=1.29960, tap_ratio=1, s_nom=1600) +network.add("Line","LuebeckSiems", bus0="26387",bus1="Siems220", x=0.0001, s_nom=1600) + + +#network.lines.s_nom = network.lines.s_nom*1.5 +#network.transformers.s_nom = network.transformers.s_nom*3 + +network.generators.control="PV" + +network.buses['v_nom'] = 380. + +trafo_index = network.transformers.index + +network.import_components_from_dataframe( + network.transformers.loc[:,['bus0','bus1','x','s_nom']] + .assign(x=0.1*380**2/2000) + .set_index('T' + trafo_index), + 'Line' +) + +network.transformers.drop(trafo_index, inplace=True) +for attr in network.transformers_t: + network.transformers_t[attr] = network.transformers_t[attr].reindex(columns=[]) + +busmap = busmap_by_kmeans(network, bus_weightings=pd.Series(np.repeat(1, len(network.buses)), index=network.buses.index) , n_clusters= 10) + + +clustering = get_clustering_from_busmap(network, busmap) +network = clustering.network +#network = cluster_on_extra_high_voltage(network, busmap, with_time=True) + + +# start powerflow calculat#ions +network.lopf(snapshots, solver_name='gurobi') + +#network.model.write('/home/ulf/file.lp', io_options={'symbolic_solver_labels':True}) + +# make a line loading plot +plot_line_loading(network) + +#plot stacked sum of nominal power for each generator type and timestep +#plot_stacked_gen(network, resolution="MW") + + +# close session +#session.close() From 753cc7f0d0dfebb653ac1235ad985bdf0180d363 Mon Sep 17 00:00:00 2001 From: ulf Date: Thu, 1 Jun 2017 15:54:17 +0200 Subject: [PATCH 06/65] add todo --- etrago/k_means_testing.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/etrago/k_means_testing.py b/etrago/k_means_testing.py index f7488720..57cf9e9e 100644 --- a/etrago/k_means_testing.py +++ b/etrago/k_means_testing.py @@ -109,6 +109,12 @@ network.buses['v_nom'] = 380. +# TODO adjust the x of the lines which are not 380. problem our lines have no v_nom. this is implicitly defined by the connected buses. Generally it should look something like the following: +#lines_v_nom_b = network.lines.v_nom != 380 +#network.lines.loc[lines_v_nom_b, 'x'] *= (380./network.lines.loc[lines_v_nom_b, 'v_nom'])**2 +#network.lines.loc[lines_v_nom_b, 'v_nom'] = 380. + + trafo_index = network.transformers.index network.import_components_from_dataframe( From 77b3e9afd86db96c973682b5e58bca96f40f0683 Mon Sep 17 00:00:00 2001 From: ulf Date: Thu, 1 Jun 2017 16:39:34 +0200 Subject: [PATCH 07/65] minor changes --- etrago/k_means_testing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etrago/k_means_testing.py b/etrago/k_means_testing.py index 57cf9e9e..3fb79ad8 100644 --- a/etrago/k_means_testing.py +++ b/etrago/k_means_testing.py @@ -128,7 +128,7 @@ for attr in network.transformers_t: network.transformers_t[attr] = network.transformers_t[attr].reindex(columns=[]) -busmap = busmap_by_kmeans(network, bus_weightings=pd.Series(np.repeat(1, len(network.buses)), index=network.buses.index) , n_clusters= 10) +busmap = busmap_by_kmeans(network, bus_weightings=pd.Series(np.repeat(1, len(network.buses)), index=network.buses.index) , n_clusters= 50) clustering = get_clustering_from_busmap(network, busmap) From fe75967367451a4746587d3c95714c36881452ed Mon Sep 17 00:00:00 2001 From: ulf Date: Fri, 2 Jun 2017 11:13:34 +0200 Subject: [PATCH 08/65] create first half-working version --- etrago/extra_high_voltage_SH_appl.py | 23 +++++------------------ 1 file changed, 5 insertions(+), 18 deletions(-) diff --git a/etrago/extra_high_voltage_SH_appl.py b/etrago/extra_high_voltage_SH_appl.py index 7987cb1e..e174baee 100644 --- a/etrago/extra_high_voltage_SH_appl.py +++ b/etrago/extra_high_voltage_SH_appl.py @@ -25,7 +25,7 @@ session = oedb_session() -scenario = 'SH Status Quo' +scenario = 'NEP 2035' # define relevant tables of generator table pq_set_cols_1 = ['p_set'] @@ -34,8 +34,8 @@ # choose relevant parameters used in pf temp_id_set = 1 -start_h = 2300 -end_h = 2301 +start_h = 2320 +end_h = 2321 # define investigated time range timerange = get_timerange(session, temp_id_set, TempResolution, start_h, end_h) @@ -104,7 +104,7 @@ #network.lines.s_nom = network.lines.s_nom*1.5 #network.transformers.s_nom = network.transformers.s_nom*1.5 - +load_shedding(network) network.generators.control="PV" @@ -115,22 +115,9 @@ #load shedding in order to hunt infeasibilities #load_shedding(network) -# Variables -#model.X = Var( model .P) - - - def extra_functionality(network,snapshots): - - network.model.objective = network.model.objective.add(expr=network.model.passive_branch_p['Line'], index=None) -# def line_loading(model,snapshot): -# -# return model.passive_branch_p['line',snapshot] - -# #add an additional objective function in order to minimize line loading -# network.model.objective.add(expr=line_loading, index=None) -# #network.model.line_loading = Objective(list(snapshots),rule=line_loading) + network.model.objective.expr += 0.01* sum(network.model.passive_branch_p[i] for i in network.model.passive_branch_p_index) # start powerflow calculations network.lopf(snapshots, solver_name='gurobi', extra_functionality=extra_functionality) From 13991d05e45fcd065438090961cef98a07abe1b5 Mon Sep 17 00:00:00 2001 From: ulfmueller Date: Tue, 4 Jul 2017 08:55:54 +0200 Subject: [PATCH 09/65] Update requirements.txt --- requirements.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/requirements.txt b/requirements.txt index 3598ebd9..0b119914 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,11 +2,11 @@ # for network clustering -# ego powerflow repo on branch upgrade-to-pypsa-0.8: https://github.com/openego/ego.powerflow/tree/refactor/upgrade-to-pypsa-0.8 --e git+https://github.com/openego/ego.powerflow.git/refactor/upgrade-to-pypsa-0.8egg=ego.powerflow +# ego powerflow repo on branch dev: +-e git+https://github.com/openego/ego.powerflow.git@dev#egg=ego.powerflow -# ego-io repo on branch upgrade-to-pypsa-0.8: https://github.com/openego/ego.io/tree/refactor/upgrade-to-pypsa-0.8 --e git+https://github.com/openego/ego.io.git@/refactor/upgrade-to-pypsa-0.8#egg=ego.io +# ego.io release 0.2.0 +egoio==0.2.0 # eGo PyPSA fork on network-clustering https://github.com/openego/PyPSA/tree/features/network-clustering --e git+https://github.com/openego/PyPSA.git@features/network-clustering#egg=PyPSA +-e git+https://github.com/openego/PyPSA.git@dev#egg=PyPSA From 3f6a556e17a02e4c6d9bf5a2020336f20072708e Mon Sep 17 00:00:00 2001 From: ulf Date: Wed, 5 Jul 2017 11:19:40 +0200 Subject: [PATCH 10/65] added extra functionality line loading minimization --- etrago/appl.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/etrago/appl.py b/etrago/appl.py index c40a092d..8dd8db42 100644 --- a/etrago/appl.py +++ b/etrago/appl.py @@ -82,6 +82,10 @@ busmap = busmap_from_psql(network, session, scn_name=args['scn_name']) network = cluster_on_extra_high_voltage(network, busmap, with_time=True) +def extra_functionality(network,snapshots): + + network.model.objective.expr += 0.01* sum(network.model.passive_branch_p[i] for i in network.model.passive_branch_p_index) + # start powerflow calculations x = time.time() network.lopf(scenario.timeindex, solver_name=args['solver']) From 914c0f9c8e03e4731eee79d7fb41d4a0b7f0a248 Mon Sep 17 00:00:00 2001 From: ulf Date: Wed, 5 Jul 2017 11:20:13 +0200 Subject: [PATCH 11/65] deleted obsolete file --- etrago/extra_high_voltage_SH_appl.py | 137 --------------------------- 1 file changed, 137 deletions(-) delete mode 100644 etrago/extra_high_voltage_SH_appl.py diff --git a/etrago/extra_high_voltage_SH_appl.py b/etrago/extra_high_voltage_SH_appl.py deleted file mode 100644 index e174baee..00000000 --- a/etrago/extra_high_voltage_SH_appl.py +++ /dev/null @@ -1,137 +0,0 @@ -"""This is the docstring for the example.py module. Modules names should -have short, all-lowercase names. The module name may have underscores if -this improves readability. -Every module should have a docstring at the very top of the file. The -module's docstring may extend over multiple lines. If your docstring does -extend over multiple lines, the closing three quotation marks must be on -a line by itself, preferably preceded by a blank line.""" - -__copyright__ = "tba" -__license__ = "tba" -__author__ = "tba" - - -from egopowerflow.tools.tools import oedb_session -from egopowerflow.tools.io import get_timerange, import_components, import_pq_sets,\ - add_source_types, create_powerflow_problem -from egopowerflow.tools.plot import add_coordinates, plot_line_loading,\ - plot_stacked_gen, curtailment, gen_dist -from egoio.db_tables.model_draft import EgoGridPfHvBus as Bus, EgoGridPfHvLine as Line, EgoGridPfHvGenerator as Generator, EgoGridPfHvLoad as Load,\ - EgoGridPfHvTransformer as Transformer, EgoGridPfHvTempResolution as TempResolution, EgoGridPfHvGeneratorPqSet as GeneratorPqSet,\ - EgoGridPfHvLoadPqSet as LoadPqSet, EgoGridPfHvSource as Source -from cluster.networkclustering import busmap_from_psql, cluster_on_extra_high_voltage -from extras.utilities import load_shedding -from pyomo.environ import Objective, Var - -session = oedb_session() - -scenario = 'NEP 2035' - -# define relevant tables of generator table -pq_set_cols_1 = ['p_set'] -pq_set_cols_2 = ['q_set'] -p_max_pu = ['p_max_pu'] - -# choose relevant parameters used in pf -temp_id_set = 1 -start_h = 2320 -end_h = 2321 - -# define investigated time range -timerange = get_timerange(session, temp_id_set, TempResolution, start_h, end_h) - -# define relevant tables -tables = [Bus, Line, Generator, Load, Transformer] - -# get components from database tables -components = import_components(tables, session, scenario) - -# create PyPSA powerflow problem -network, snapshots = create_powerflow_problem(timerange, components) - -# import pq-set tables to pypsa network (p_set for generators and loads) -pq_object = [GeneratorPqSet, LoadPqSet] -network = import_pq_sets(session=session, - network=network, - pq_tables=pq_object, - timerange=timerange, - scenario=scenario, - columns=pq_set_cols_1, - start_h=start_h, - end_h=end_h) - -# import pq-set table to pypsa network (q_set for loads) -network = import_pq_sets(session=session, - network=network, - pq_tables=[LoadPqSet], - timerange=timerange, - scenario=scenario, - columns=pq_set_cols_2, - start_h=start_h, - end_h=end_h) - -network = import_pq_sets(session=session, - network=network, - pq_tables=[GeneratorPqSet], - timerange=timerange, - scenario=scenario, - columns=p_max_pu, - start_h=start_h, - end_h=end_h) - -## import time data for storages: -#network = import_pq_sets(session=session, -# network=network, -# pq_tables=[StoragePqSet], -# timerange=timerange, -# scenario=scenario, -# columns=storage_sets, -# start_h=start_h, -# end_h=end_h) - -# add coordinates to network nodes and make ready for map plotting -network = add_coordinates(network) - -# add source names to generators -add_source_types(session, network, table=Source) - -#add connection from Luebeck to Siems -network.add("Bus", "Siems220",carrier='AC', v_nom=220, x=10.760835, y=53.909745) -network.add("Transformer", "Siems220_380", bus0="25536", bus1="Siems220", x=1.29960, tap_ratio=1, s_nom=1600) -network.add("Line","LuebeckSiems", bus0="26387",bus1="Siems220", x=0.0001, s_nom=1600) - - -#network.lines.s_nom = network.lines.s_nom*1.5 -#network.transformers.s_nom = network.transformers.s_nom*1.5 - -load_shedding(network) - -network.generators.control="PV" - -#busmap = busmap_from_psql(network, session, scn_name=scenario) - -#network = cluster_on_extra_high_voltage(network, busmap, with_time=True) - -#load shedding in order to hunt infeasibilities -#load_shedding(network) - -def extra_functionality(network,snapshots): - - network.model.objective.expr += 0.01* sum(network.model.passive_branch_p[i] for i in network.model.passive_branch_p_index) - -# start powerflow calculations -network.lopf(snapshots, solver_name='gurobi', extra_functionality=extra_functionality) - -network.model.write('/home/ulf/file.lp', io_options={'symbolic_solver_labels':True}) - -# make a line loading plot -plot_line_loading(network) - -#plot stacked sum of nominal power for each generator type and timestep -plot_stacked_gen(network, resolution="MW") - -# same as before, limited to one specific bus -plot_stacked_gen(network, bus='24560', resolution='MW') - -# close session -session.close() From 9a1aa7043b30dc00f7d51a7863323c851c5d3ef0 Mon Sep 17 00:00:00 2001 From: ulf Date: Wed, 5 Jul 2017 11:22:44 +0200 Subject: [PATCH 12/65] changed appl.py naming --- etrago/{appl.py => line_loading_minimization_appl.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename etrago/{appl.py => line_loading_minimization_appl.py} (100%) diff --git a/etrago/appl.py b/etrago/line_loading_minimization_appl.py similarity index 100% rename from etrago/appl.py rename to etrago/line_loading_minimization_appl.py From a3f421e04963267851d3304065045fbb4909d745 Mon Sep 17 00:00:00 2001 From: ulf Date: Mon, 10 Jul 2017 17:31:15 +0200 Subject: [PATCH 13/65] line x adjustment implemented --- etrago/appl.py | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/etrago/appl.py b/etrago/appl.py index f32d01d2..52f13368 100644 --- a/etrago/appl.py +++ b/etrago/appl.py @@ -20,6 +20,9 @@ storage_distribution) from extras.utilities import load_shedding, data_manipulation_sh, results_to_csv from cluster.networkclustering import busmap_from_psql, cluster_on_extra_high_voltage +from pypsa.networkclustering import busmap_by_kmeans, get_clustering_from_busmap +import pandas as pd + args = {'network_clustering':False, 'db': 'oedb', # db session @@ -84,6 +87,42 @@ busmap = busmap_from_psql(network, session, scn_name=args['scn_name']) network = cluster_on_extra_high_voltage(network, busmap, with_time=True) +# k-means clustering (first try) + +network.generators.control="PV" + +network.buses['v_nom'] = 380. + +#problem our lines have no v_nom. this is implicitly defined by the connected buses: +network.lines["v_nom"] = network.lines.bus0.map(network.buses.v_nom) + +# adjust the x of the lines which are not 380. +lines_v_nom_b = network.lines.v_nom != 380 +network.lines.loc[lines_v_nom_b, 'x'] *= (380./network.lines.loc[lines_v_nom_b, 'v_nom'])**2 +network.lines.loc[lines_v_nom_b, 'v_nom'] = 380. + + +trafo_index = network.transformers.index + +network.import_components_from_dataframe( + network.transformers.loc[:,['bus0','bus1','x','s_nom']] + .assign(x=0.1*380**2/2000) + .set_index('T' + trafo_index), + 'Line' +) + +network.transformers.drop(trafo_index, inplace=True) +for attr in network.transformers_t: + network.transformers_t[attr] = network.transformers_t[attr].reindex(columns=[]) + +busmap = busmap_by_kmeans(network, bus_weightings=pd.Series(np.repeat(1, len(network.buses)), index=network.buses.index) , n_clusters= 50) + + +clustering = get_clustering_from_busmap(network, busmap) +network = clustering.network +#network = cluster_on_extra_high_voltage(network, busmap, with_time=True) + + # start powerflow calculations if args['method'] == 'lopf': x = time.time() From 9ecf4e90f75c529ddef773fb84cf564b44b45647 Mon Sep 17 00:00:00 2001 From: mariusves Date: Wed, 12 Jul 2017 12:50:17 +0200 Subject: [PATCH 14/65] add export2db function --- etrago/appl.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/etrago/appl.py b/etrago/appl.py index 7ebce755..9442da4b 100644 --- a/etrago/appl.py +++ b/etrago/appl.py @@ -13,7 +13,7 @@ import numpy as np np.random.seed() from egopowerflow.tools.tools import oedb_session -from egopowerflow.tools.io import NetworkScenario +from egopowerflow.tools.io import NetworkScenario, results_to_oedb import time from egopowerflow.tools.plot import (plot_line_loading, plot_stacked_gen, add_coordinates, curtailment, gen_dist, @@ -31,11 +31,13 @@ 'ormcls_prefix': 'EgoGridPfHv', #if gridversion:'version-number' then 'EgoPfHv', if gridversion:None then 'EgoGridPfHv' 'lpfile': False, # state if and where you want to save pyomo's lp file: False or '/path/tofolder' 'results':False , # state if and where you want to save results as csv: False or '/path/tofolder' + 'export': False, # state if you want to export the results back to the database 'solver': 'gurobi', #glpk, cplex or gurobi 'branch_capacity_factor': 1, #to globally extend or lower branch capacities 'storage_extendable':True, 'load_shedding':True, - 'generator_noise':False} + 'generator_noise':False, + 'comments': ''} session = oedb_session(args['db']) @@ -95,9 +97,14 @@ if not args['lpfile'] == False: network.model.write(args['lpfile'], io_options={'symbolic_solver_labels': True}) + +# write PyPSA results back to database +if args['export']: + results_to_oedb(session, network, 'hv', args) + # write PyPSA results to csv to path results_to_csv(network, args['results']) - + # plots # make a line loading plot From 9a5faea3081e363c6e69ba945ff079a651eb9d33 Mon Sep 17 00:00:00 2001 From: MariusVes Date: Thu, 13 Jul 2017 12:12:44 +0200 Subject: [PATCH 15/65] Fix for the naming of the new bus/trafo/line In the function data_manipulation_sh() the newly added components don't use integers as IDs which can cause problems with other future functions (e.g. export result back to the database). Therefore, the new components will get a new unique ID which is being generated from the highest last used ID in the available data set. --- etrago/extras/utilities.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/etrago/extras/utilities.py b/etrago/extras/utilities.py index 9d57a72e..d1cbac88 100644 --- a/etrago/extras/utilities.py +++ b/etrago/extras/utilities.py @@ -126,9 +126,12 @@ def load_shedding (network, **kwargs): def data_manipulation_sh (network): #add connection from Luebeck to Siems - network.add("Bus", "Siems220",carrier='AC', v_nom=220, x=10.760835, y=53.909745) - network.add("Transformer", "Siems220_380", bus0="25536", bus1="Siems220", x=1.29960, tap_ratio=1, s_nom=1600) - network.add("Line","LuebeckSiems", bus0="26387",bus1="Siems220", x=0.0001, s_nom=1600) + new_bus = str(int(network.buses.index.max())+1) + new_trafo = str(int(network.transformers.index.max())+1) + new_line = str(int(network.lines.index.max())+1) + network.add("Bus", new_bus,carrier='AC', v_nom=220, x=10.760835, y=53.909745) + network.add("Transformer", new_trafo, bus0="25536", bus1=new_bus, x=1.29960, tap_ratio=1, s_nom=1600) + network.add("Line",new_line, bus0="26387",bus1=new_bus, x=0.0001, s_nom=1600) return From 13db7360d503a5c603f7783b97b6c1a9c49566a4 Mon Sep 17 00:00:00 2001 From: kimvk Date: Tue, 18 Jul 2017 09:51:44 +0200 Subject: [PATCH 16/65] Add constraints to extra-functionaliy --- etrago/line_loading_minimization_appl.py | 31 +++++++++++++++++++----- 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/etrago/line_loading_minimization_appl.py b/etrago/line_loading_minimization_appl.py index 8dd8db42..afc5299c 100644 --- a/etrago/line_loading_minimization_appl.py +++ b/etrago/line_loading_minimization_appl.py @@ -14,6 +14,9 @@ np.random.seed() from egopowerflow.tools.tools import oedb_session from egopowerflow.tools.io import NetworkScenario +from pyomo.environ import Constraint +from pyomo.environ import Var +from pyomo.environ import NonNegativeIntegers import time from egopowerflow.tools.plot import plot_line_loading, plot_stacked_gen, add_coordinates, curtailment, gen_dist, storage_distribution from extras.utilities import load_shedding, data_manipulation_sh, results_to_csv @@ -24,7 +27,7 @@ 'gridversion':None, #None for model_draft or Version number (e.g. v0.2.10) for grid schema 'method': 'lopf', # lopf or pf 'start_h': 2301, - 'end_h' : 2302, + 'end_h' : 2303, 'scn_name': 'SH Status Quo', 'ormcls_prefix': 'EgoGridPfHv', #if gridversion:'version-number' then 'EgoPfHv', if gridversion:None then 'EgoGridPfHv' 'outfile': '/path', # state if and where you want to save pyomo's lp file @@ -83,18 +86,34 @@ network = cluster_on_extra_high_voltage(network, busmap, with_time=True) def extra_functionality(network,snapshots): + - network.model.objective.expr += 0.01* sum(network.model.passive_branch_p[i] for i in network.model.passive_branch_p_index) - + network.model.Hilfzahl1= Var(within = NonNegativeIntegers) + network.model.Hilfzahl2= Var(within = NonNegativeIntegers) + + def cRule(model,snapshots): + for i in network.model.passive_branch_p_index: + network.model.passive_branch_p[i] == network.model.Hilfzahl1 - network.model.Hilfzahl2 + return (network.model.passive_branch_p[i]) + network.model.cRule=Constraint(list(snapshots),rule=cRule) + +#============================================================================== +# def cRule(model,snapshots): +# return (network.model.passive_branch_p[i] for i in network.model.passive_branch_p_index == network.model.Hilfzahl1 - network.model.Hilfzahl2) +# network.model.cRule=Constraint(list(snapshots),rule=cRule) +#============================================================================== + + network.model.objective.expr += 0.01* sum(network.model.passive_branch_p[i] for i in network.model.passive_branch_p_index) + # start powerflow calculations x = time.time() -network.lopf(scenario.timeindex, solver_name=args['solver']) +network.lopf(scenario.timeindex, solver_name=args['solver'], extra_functionality=extra_functionality) y = time.time() z = (y - x) / 60 # z is time for lopf in minutes # write results -network.model.write(args['outfile'], io_options={'symbolic_solver_labels': - True}) +#network.model.write(args['outfile'], io_options={'symbolic_solver_labels': +# True}) results_to_csv(network, args['results']) # plots From 436aa9f3bb3b89257c43a0ae6649f803eb564157 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 3 Aug 2017 14:16:03 +0200 Subject: [PATCH 17/65] Modified Extra-Functionality --- etrago/line_loading_minimization_appl.py | 44 ++++++++++-------------- 1 file changed, 19 insertions(+), 25 deletions(-) diff --git a/etrago/line_loading_minimization_appl.py b/etrago/line_loading_minimization_appl.py index afc5299c..5bd19520 100644 --- a/etrago/line_loading_minimization_appl.py +++ b/etrago/line_loading_minimization_appl.py @@ -14,9 +14,7 @@ np.random.seed() from egopowerflow.tools.tools import oedb_session from egopowerflow.tools.io import NetworkScenario -from pyomo.environ import Constraint -from pyomo.environ import Var -from pyomo.environ import NonNegativeIntegers +from pyomo.environ import (Var,Constraint, PositiveReals,ConcreteModel) import time from egopowerflow.tools.plot import plot_line_loading, plot_stacked_gen, add_coordinates, curtailment, gen_dist, storage_distribution from extras.utilities import load_shedding, data_manipulation_sh, results_to_csv @@ -27,7 +25,7 @@ 'gridversion':None, #None for model_draft or Version number (e.g. v0.2.10) for grid schema 'method': 'lopf', # lopf or pf 'start_h': 2301, - 'end_h' : 2303, + 'end_h' : 2303, 'scn_name': 'SH Status Quo', 'ormcls_prefix': 'EgoGridPfHv', #if gridversion:'version-number' then 'EgoPfHv', if gridversion:None then 'EgoGridPfHv' 'outfile': '/path', # state if and where you want to save pyomo's lp file @@ -85,35 +83,31 @@ busmap = busmap_from_psql(network, session, scn_name=args['scn_name']) network = cluster_on_extra_high_voltage(network, busmap, with_time=True) -def extra_functionality(network,snapshots): +def extra_functionality(network,snapshots): - - network.model.Hilfzahl1= Var(within = NonNegativeIntegers) - network.model.Hilfzahl2= Var(within = NonNegativeIntegers) - - def cRule(model,snapshots): - for i in network.model.passive_branch_p_index: - network.model.passive_branch_p[i] == network.model.Hilfzahl1 - network.model.Hilfzahl2 - return (network.model.passive_branch_p[i]) - network.model.cRule=Constraint(list(snapshots),rule=cRule) - -#============================================================================== -# def cRule(model,snapshots): -# return (network.model.passive_branch_p[i] for i in network.model.passive_branch_p_index == network.model.Hilfzahl1 - network.model.Hilfzahl2) -# network.model.cRule=Constraint(list(snapshots),rule=cRule) -#============================================================================== - - network.model.objective.expr += 0.01* sum(network.model.passive_branch_p[i] for i in network.model.passive_branch_p_index) + network.model.number1= Var(network.model.passive_branch_p_index, within = PositiveReals) + network.model.number2= Var(network.model.passive_branch_p_index, within = PositiveReals) + + def cRule(model): + for i in network.model.passive_branch_p_index: + return (model.number1[i] + model.number2[i] == network.model.passive_branch_p[i]) + network.model.cRule=Constraint(rule=cRule) + + network.model.objective.expr += 0.01* sum(network.model.passive_branch_p[i] for i in network.model.passive_branch_p_index) # start powerflow calculations x = time.time() -network.lopf(scenario.timeindex, solver_name=args['solver'], extra_functionality=extra_functionality) +network.lopf(scenario.timeindex,solver_name=args['solver'], + extra_functionality=extra_functionality) y = time.time() + z = (y - x) / 60 # z is time for lopf in minutes + + # write results -#network.model.write(args['outfile'], io_options={'symbolic_solver_labels': -# True}) +# network.model.write(args['outfile'], io_options={'symbolic_solver_labels':True}) + results_to_csv(network, args['results']) # plots From d527d96d5b032fcbe24d933758dc4985f9468b7c Mon Sep 17 00:00:00 2001 From: mariusves Date: Thu, 17 Aug 2017 15:54:38 +0200 Subject: [PATCH 18/65] fix data_manipulation_sh added geoms/topos and fixed bugged index generation --- .gitignore | 1 + etrago/extras/utilities.py | 30 ++++++++++++++++++++++++------ 2 files changed, 25 insertions(+), 6 deletions(-) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..b021e7b1 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +etrago/cluster/__pycache__/networkclustering.cpython-36.pyc \ No newline at end of file diff --git a/etrago/extras/utilities.py b/etrago/extras/utilities.py index 9d57a72e..59021cd0 100644 --- a/etrago/extras/utilities.py +++ b/etrago/extras/utilities.py @@ -116,19 +116,37 @@ def load_shedding (network, **kwargs): p_nom=p_nom, carrier='load shedding', bus=network.buses.index), - index=network.buses.index + ' load'), + index=network.buses.index + '00000'), "Generator" ) - return def data_manipulation_sh (network): - + from shapely.geometry import Point, LineString, MultiLineString + from geoalchemy2.shape import from_shape, to_shape + + new_bus = str(network.buses.index.astype(int).max()+1) + new_trafo = str(network.transformers.index.astype(int).max()+1) + new_line = str(network.lines.index.astype(int).max()+1) + #add connection from Luebeck to Siems - network.add("Bus", "Siems220",carrier='AC', v_nom=220, x=10.760835, y=53.909745) - network.add("Transformer", "Siems220_380", bus0="25536", bus1="Siems220", x=1.29960, tap_ratio=1, s_nom=1600) - network.add("Line","LuebeckSiems", bus0="26387",bus1="Siems220", x=0.0001, s_nom=1600) + network.add("Bus", new_bus,carrier='AC', v_nom=220, x=10.760835, y=53.909745) + network.add("Transformer", new_trafo, bus0="25536", bus1=new_bus, x=1.29960, tap_ratio=1, s_nom=1600) + network.add("Line",new_line, bus0="26387",bus1=new_bus, x=0.0001, s_nom=1600) + network.lines.cables[new_line]=3.0 + + #bus geom + point_bus1 = Point(10.760835,53.909745) + network.buses.geom[new_bus]= from_shape(point_bus1, 4326) + + #line geom/topo + network.lines.geom[new_line] = from_shape(MultiLineString([LineString([to_shape(network.buses.geom['26387']),point_bus1])]),4326) + network.lines.topo[new_line] = from_shape(LineString([to_shape(network.buses.geom['26387']),point_bus1]),4326) + + #trafo geom/topo + network.transformers.geom[new_trafo] = from_shape(MultiLineString([LineString([to_shape(network.buses.geom['25536']),point_bus1])]),4326) + network.transformers.topo[new_trafo] = from_shape(LineString([to_shape(network.buses.geom['25536']),point_bus1]),4326) return From d9f5d09d2dcec2b6df485d95437398669c7d71a2 Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 18 Aug 2017 14:43:47 +0200 Subject: [PATCH 19/65] Modified Extra_Functionality --- etrago/line_loading_minimization_appl.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/etrago/line_loading_minimization_appl.py b/etrago/line_loading_minimization_appl.py index 5bd19520..06e2f3c3 100644 --- a/etrago/line_loading_minimization_appl.py +++ b/etrago/line_loading_minimization_appl.py @@ -90,11 +90,11 @@ def extra_functionality(network,snapshots): def cRule(model): for i in network.model.passive_branch_p_index: - return (model.number1[i] + model.number2[i] == network.model.passive_branch_p[i]) + return (network.model.number1[i] + network.model.number2[i] == network.model.passive_branch_p[i]) network.model.cRule=Constraint(rule=cRule) network.model.objective.expr += 0.01* sum(network.model.passive_branch_p[i] for i in network.model.passive_branch_p_index) - + # start powerflow calculations x = time.time() network.lopf(scenario.timeindex,solver_name=args['solver'], @@ -106,7 +106,7 @@ def cRule(model): # write results -# network.model.write(args['outfile'], io_options={'symbolic_solver_labels':True}) +network.model.write(args['outfile'], io_options={'symbolic_solver_labels':True}) results_to_csv(network, args['results']) From d93bf19641d6b65841edda56143df75d81610561 Mon Sep 17 00:00:00 2001 From: Martin Date: Fri, 18 Aug 2017 15:29:12 +0200 Subject: [PATCH 20/65] Update extra_functionality --- etrago/line_loading_minimization_appl.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/etrago/line_loading_minimization_appl.py b/etrago/line_loading_minimization_appl.py index 06e2f3c3..cacb7f5c 100644 --- a/etrago/line_loading_minimization_appl.py +++ b/etrago/line_loading_minimization_appl.py @@ -83,16 +83,16 @@ busmap = busmap_from_psql(network, session, scn_name=args['scn_name']) network = cluster_on_extra_high_voltage(network, busmap, with_time=True) -def extra_functionality(network,snapshots): - - network.model.number1= Var(network.model.passive_branch_p_index, within = PositiveReals) - network.model.number2= Var(network.model.passive_branch_p_index, within = PositiveReals) - - def cRule(model): - for i in network.model.passive_branch_p_index: - return (network.model.number1[i] + network.model.number2[i] == network.model.passive_branch_p[i]) - network.model.cRule=Constraint(rule=cRule) - +def extra_functionality(network,snapshots): + + network.model.number1 = Var(network.model.passive_branch_p_index, within = PositiveReals) + network.model.number2 = Var(network.model.passive_branch_p_index, within = PositiveReals) + + def cRule(model, c, l, t): + return (model.number1[c, l, t] + model.number2[c, l, t] == model.passive_branch_p[c, l, t]) + + network.model.cRule=Constraint(network.model.passive_branch_p_index, rule=cRule) + network.model.objective.expr += 0.01* sum(network.model.passive_branch_p[i] for i in network.model.passive_branch_p_index) # start powerflow calculations From 8dddb4acfc71c5a3f91509b7f14f9930db687c7d Mon Sep 17 00:00:00 2001 From: Lukas Date: Mon, 21 Aug 2017 13:45:39 +0200 Subject: [PATCH 21/65] extend only extendable storages --- etrago/appl.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/etrago/appl.py b/etrago/appl.py index a0e5182c..9a472ada 100644 --- a/etrago/appl.py +++ b/etrago/appl.py @@ -24,19 +24,19 @@ args = {'network_clustering':False, 'db': 'oedb', # db session - 'gridversion':None, #None for model_draft or Version number (e.g. v0.2.10) for grid schema + 'gridversion': None, #None for model_draft or Version number (e.g. v0.2.10) for grid schema 'method': 'lopf', # lopf or pf - 'start_h': 2320, - 'end_h' : 2324, - 'scn_name': 'SH Status Quo', + 'start_h': 1, + 'end_h' : 2, + 'scn_name': 'Status Quo', 'ormcls_prefix': 'EgoGridPfHv', #if gridversion:'version-number' then 'EgoPfHv', if gridversion:None then 'EgoGridPfHv' 'lpfile': False, # state if and where you want to save pyomo's lp file: False or '/path/tofolder' - 'results': False , # state if and where you want to save results as csv: False or '/path/tofolder' + 'results': False, # state if and where you want to save results as csv: False or '/path/tofolder' 'solver': 'gurobi', #glpk, cplex or gurobi 'branch_capacity_factor': 1, #to globally extend or lower branch capacities 'storage_extendable':True, - 'load_shedding':True, - 'generator_noise':False, + 'load_shedding':False, + 'generator_noise':True, 'parallelisation':False} def etrago(args): @@ -55,7 +55,7 @@ def etrago(args): # add coordinates network = add_coordinates(network) - + # create generator noise noise_values = network.generators.marginal_cost + abs(np.random.normal(0,0.001,len(network.generators.marginal_cost))) np.savetxt("noise_values.csv", noise_values, delimiter=",") @@ -65,18 +65,17 @@ def etrago(args): network.lines.s_nom = network.lines.s_nom*args['branch_capacity_factor'] network.transformers.s_nom = network.transformers.s_nom*args['branch_capacity_factor'] - if args['generator_noise']: # add random noise to all generators with marginal_cost of 0. network.generators.marginal_cost = noise_values if args['storage_extendable']: # set virtual storages to be extendable - network.storage_units.p_nom_extendable = True + if network.storage_units.source.any()=='extendable_storage': + network.storage_units.p_nom_extendable = True # set virtual storage costs with regards to snapshot length - network.storage_units.capital_cost = (network.storage_units.capital_cost / - (8760//(args['end_h']-args['start_h']+1))) - + network.storage_units.capital_cost = (network.storage_units.capital_cost / + (8760//(args['end_h']-args['start_h']+1))) # for SH scenario run do data preperation: if args['scn_name'] == 'SH Status Quo': @@ -125,7 +124,8 @@ def etrago(args): plot_stacked_gen(network, resolution="MW") # plot to show extendable storages -#storage_distribution(network) +storage_distribution(network) # close session #session.close() + From 0cbbe6a8e1ae4be1bda4610ff106670a44cc2c6c Mon Sep 17 00:00:00 2001 From: wolfbunke Date: Tue, 22 Aug 2017 15:44:15 +0200 Subject: [PATCH 22/65] add new structure due to changes in dev --- etrago/appl.py | 8 ++--- etrago/k_means_testing.py | 67 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+), 4 deletions(-) create mode 100644 etrago/k_means_testing.py diff --git a/etrago/appl.py b/etrago/appl.py index 91b96f72..b2be943f 100644 --- a/etrago/appl.py +++ b/etrago/appl.py @@ -21,7 +21,7 @@ storage_distribution) from etrago.extras.utilities import load_shedding, data_manipulation_sh, results_to_csv, parallelisation, pf_post_lopf from etrago.cluster.networkclustering import busmap_from_psql, cluster_on_extra_high_voltage - +""" args = {'network_clustering':False, 'db': 'oedb', # db session 'gridversion': None, #None for model_draft or Version number (e.g. v0.2.10) for grid schema @@ -39,7 +39,7 @@ 'load_shedding':True, 'generator_noise':True, 'parallelisation':False} - +""" def etrago(args): session = oedb_session(args['db']) @@ -120,7 +120,7 @@ def etrago(args): return network - +""" # execute etrago function network = etrago(args) @@ -134,7 +134,7 @@ def etrago(args): # plot to show extendable storages storage_distribution(network) - +""" # close session #session.close() diff --git a/etrago/k_means_testing.py b/etrago/k_means_testing.py new file mode 100644 index 00000000..62776699 --- /dev/null +++ b/etrago/k_means_testing.py @@ -0,0 +1,67 @@ +""" +K-means testing File + +see: https://github.com/openego/eTraGo/issues/6 + +ToDo's: +------- +the remaining work would be: + +- [x] implement the [todo](https://github.com/openego/eTraGo/blob/features/k-means-clustering/etrago/k_means_testing.py#L112-L115) so that the x of the lines which are newly defined as 380kV lines are adjusted + +- [ ] in the [Hoersch and Brown contribution](https://arxiv.org/pdf/1705.07617.pdf) in Chapter II 2) and follwoing the weighting is defined. the weighting right now is equal over all buses. This should be changed to the assumptions with respect to the contribution or define another senseful weighting + +- [ ] add functionality to save the resulting cluster for reproducibility + +- [ ] convert it to a function and move it [here](https://github.com/openego/eTraGo/blob/features/k-means-clustering/etrago/cluster/networkclustering.py) + + +""" + +__copyright__ = "tba" +__license__ = "tba" +__author__ = "tba" + +from appl import etrago +import json + +# import scenario settings +with open('scenario_setting.json') as f: + scenario_setting = json.load(f) + + + +test = etrago(scenario_setting) + + + +""" +network.buses['v_nom'] = 380. + +# TODO adjust the x of the lines which are not 380. problem our lines have no v_nom. this is implicitly defined by the connected buses. Generally it should look something like the following: +#lines_v_nom_b = network.lines.v_nom != 380 +#network.lines.loc[lines_v_nom_b, 'x'] *= (380./network.lines.loc[lines_v_nom_b, 'v_nom'])**2 +#network.lines.loc[lines_v_nom_b, 'v_nom'] = 380. + + +trafo_index = network.transformers.index + +network.import_components_from_dataframe( + network.transformers.loc[:,['bus0','bus1','x','s_nom']] + .assign(x=0.1*380**2/2000) + .set_index('T' + trafo_index), + 'Line' +) + +network.transformers.drop(trafo_index, inplace=True) +for attr in network.transformers_t: + network.transformers_t[attr] = network.transformers_t[attr].reindex(columns=[]) + +busmap = busmap_by_kmeans(network, bus_weightings=pd.Series(np.repeat(1, len(network.buses)), index=network.buses.index) , n_clusters= 50) + + +clustering = get_clustering_from_busmap(network, busmap) +network = clustering.network +#network = cluster_on_extra_high_voltage(network, busmap, with_time=True) + +""" From 236b39b8c7b035327c4fb42ac3857995e1622326 Mon Sep 17 00:00:00 2001 From: wolfbunke Date: Tue, 22 Aug 2017 15:45:01 +0200 Subject: [PATCH 23/65] add args as scenario_setting json file --- etrago/scenario_setting.json | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 etrago/scenario_setting.json diff --git a/etrago/scenario_setting.json b/etrago/scenario_setting.json new file mode 100644 index 00000000..634008a2 --- /dev/null +++ b/etrago/scenario_setting.json @@ -0,0 +1,18 @@ +{ +"network_clustering":"False", +"db": "znes", +"gridversion": "None", +"method": "lopf", +"pf_post_lopf":"False", +"start_h": 2323, +"end_h" : 2324, +"scn_name": "SH Status Quo", +"ormcls_prefix": "EgoGridPfHv", +"lpfile": "False", +"results": "False", +"solver": "gurobi", +"branch_capacity_factor": 1, +"storage_extendable":"True", +"load_shedding":"True", +"generator_noise":"True", +"parallelisation":"False"} From 26b1428d1f89b4782dce5f3da019845dd61db80c Mon Sep 17 00:00:00 2001 From: wolfbunke Date: Wed, 23 Aug 2017 11:33:07 +0200 Subject: [PATCH 24/65] change json structure --- etrago/scenario_setting.json | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/etrago/scenario_setting.json b/etrago/scenario_setting.json index 634008a2..215a8a60 100644 --- a/etrago/scenario_setting.json +++ b/etrago/scenario_setting.json @@ -1,18 +1,20 @@ { -"network_clustering":"False", -"db": "znes", -"gridversion": "None", +"network_clustering": false, +"db": "oedb", +"gridversion": "v0.2.11", "method": "lopf", -"pf_post_lopf":"False", +"pf_post_lopf": false, "start_h": 2323, "end_h" : 2324, "scn_name": "SH Status Quo", -"ormcls_prefix": "EgoGridPfHv", -"lpfile": "False", -"results": "False", +"ormcls_prefix": "EgoPfHv", +"lpfile": false, +"results": false, "solver": "gurobi", "branch_capacity_factor": 1, -"storage_extendable":"True", -"load_shedding":"True", -"generator_noise":"True", -"parallelisation":"False"} +"storage_extendable": true, +"load_shedding": true, +"generator_noise": true, +"parallelisation": false +} + From ddc04ad99a3a6a49022563d8f2d12515eaf16448 Mon Sep 17 00:00:00 2001 From: wolfbunke Date: Wed, 23 Aug 2017 11:34:22 +0200 Subject: [PATCH 25/65] set settings --- etrago/appl.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/etrago/appl.py b/etrago/appl.py index b2be943f..0fee6774 100644 --- a/etrago/appl.py +++ b/etrago/appl.py @@ -24,13 +24,13 @@ """ args = {'network_clustering':False, 'db': 'oedb', # db session - 'gridversion': None, #None for model_draft or Version number (e.g. v0.2.10) for grid schema + 'gridversion': 'v0.2.11', #None for model_draft or Version number (e.g. v0.2.10) for grid schema 'method': 'lopf', # lopf or pf 'pf_post_lopf':False , #state whether you want to perform a pf after a lopf simulation 'start_h': 2323, 'end_h' : 2324, 'scn_name': 'SH Status Quo', - 'ormcls_prefix': 'EgoGridPfHv', #if gridversion:'version-number' then 'EgoPfHv', if gridversion:None then 'EgoGridPfHv' + 'ormcls_prefix': 'EgoPfHv', #if gridversion:'version-number' then 'EgoPfHv', if gridversion:None then 'EgoGridPfHv' 'lpfile': False, # state if and where you want to save pyomo's lp file: False or '/path/tofolder' 'results': False, # state if and where you want to save results as csv: False or '/path/tofolder' 'solver': 'gurobi', #glpk, cplex or gurobi @@ -119,7 +119,6 @@ def etrago(args): results_to_csv(network, args['results']) return network - """ # execute etrago function network = etrago(args) From ec6e3d6bd6b50ba2830c469df7edb9097e636b19 Mon Sep 17 00:00:00 2001 From: wolfbunke Date: Wed, 23 Aug 2017 11:40:01 +0200 Subject: [PATCH 26/65] add etrago function --- etrago/k_means_testing.py | 156 ++++++++++++++++++++++++++++---------- 1 file changed, 118 insertions(+), 38 deletions(-) diff --git a/etrago/k_means_testing.py b/etrago/k_means_testing.py index 62776699..5221b534 100644 --- a/etrago/k_means_testing.py +++ b/etrago/k_means_testing.py @@ -15,6 +15,12 @@ - [ ] convert it to a function and move it [here](https://github.com/openego/eTraGo/blob/features/k-means-clustering/etrago/cluster/networkclustering.py) +Error handling: + +* use pip3 install scikit-learn in order to use PyPSA busmap_by_kmeans + + + """ @@ -22,46 +28,120 @@ __license__ = "tba" __author__ = "tba" -from appl import etrago + +import numpy as np +from numpy import genfromtxt +np.random.seed() +from egopowerflow.tools.tools import oedb_session +from egopowerflow.tools.io import NetworkScenario +import time +from egopowerflow.tools.plot import (plot_line_loading, plot_stacked_gen, + add_coordinates, curtailment, gen_dist, + storage_distribution) +from etrago.extras.utilities import load_shedding, data_manipulation_sh, results_to_csv, parallelisation, pf_post_lopf +from etrago.cluster.networkclustering import busmap_from_psql, cluster_on_extra_high_voltage + +# imports for k-mean +from pypsa.networkclustering import busmap_by_kmeans, get_clustering_from_busmap +import pandas as pd import json -# import scenario settings +# import scenario settings **args with open('scenario_setting.json') as f: scenario_setting = json.load(f) - - -test = etrago(scenario_setting) - - - -""" -network.buses['v_nom'] = 380. - -# TODO adjust the x of the lines which are not 380. problem our lines have no v_nom. this is implicitly defined by the connected buses. Generally it should look something like the following: -#lines_v_nom_b = network.lines.v_nom != 380 -#network.lines.loc[lines_v_nom_b, 'x'] *= (380./network.lines.loc[lines_v_nom_b, 'v_nom'])**2 -#network.lines.loc[lines_v_nom_b, 'v_nom'] = 380. - - -trafo_index = network.transformers.index - -network.import_components_from_dataframe( - network.transformers.loc[:,['bus0','bus1','x','s_nom']] - .assign(x=0.1*380**2/2000) - .set_index('T' + trafo_index), - 'Line' -) - -network.transformers.drop(trafo_index, inplace=True) -for attr in network.transformers_t: - network.transformers_t[attr] = network.transformers_t[attr].reindex(columns=[]) - -busmap = busmap_by_kmeans(network, bus_weightings=pd.Series(np.repeat(1, len(network.buses)), index=network.buses.index) , n_clusters= 50) - - -clustering = get_clustering_from_busmap(network, busmap) -network = clustering.network -#network = cluster_on_extra_high_voltage(network, busmap, with_time=True) - -""" +args = scenario_setting + +def etrago(args): + session = oedb_session(args['db']) + + # additional arguments cfgpath, version, prefix + scenario = NetworkScenario(session, + version=args['gridversion'], + prefix=args['ormcls_prefix'], + method=args['method'], + start_h=args['start_h'], + end_h=args['end_h'], + scn_name=args['scn_name']) + + network = scenario.build_network() + + # add coordinates + network = add_coordinates(network) + + # TEMPORARY vague adjustment due to transformer bug in data processing + #network.transformers.x=network.transformers.x*0.01 + + + if args['branch_capacity_factor']: + network.lines.s_nom = network.lines.s_nom*args['branch_capacity_factor'] + network.transformers.s_nom = network.transformers.s_nom*args['branch_capacity_factor'] + + if args['generator_noise']: + # create generator noise + noise_values = network.generators.marginal_cost + abs(np.random.normal(0,0.001,len(network.generators.marginal_cost))) + np.savetxt("noise_values.csv", noise_values, delimiter=",") + noise_values = genfromtxt('noise_values.csv', delimiter=',') + # add random noise to all generator + network.generators.marginal_cost = noise_values + + if args['storage_extendable']: + # set virtual storages to be extendable + if network.storage_units.source.any()=='extendable_storage': + network.storage_units.p_nom_extendable = True + # set virtual storage costs with regards to snapshot length + network.storage_units.capital_cost = (network.storage_units.capital_cost / + (8760//(args['end_h']-args['start_h']+1))) + + # for SH scenario run do data preperation: + if args['scn_name'] == 'SH Status Quo': + data_manipulation_sh(network) + + #load shedding in order to hunt infeasibilities + if args['load_shedding']: + load_shedding(network) + + # network clustering + if args['network_clustering']: + network.generators.control="PV" + busmap = busmap_from_psql(network, session, scn_name=args['scn_name']) + network = cluster_on_extra_high_voltage(network, busmap, with_time=True) + + # parallisation + if args['parallelisation']: + parallelisation(network, start_h=args['start_h'], end_h=args['end_h'],group_size=1, solver_name=args['solver']) + # start linear optimal powerflow calculations + elif args['method'] == 'lopf': + x = time.time() + network.lopf(scenario.timeindex, solver_name=args['solver']) + y = time.time() + z = (y - x) / 60 # z is time for lopf in minutes + # start non-linear powerflow simulation + elif args['method'] == 'pf': + network.pf(scenario.timeindex) + if args['pf_post_lopf']: + pf_post_lopf(network, scenario) + + # write lpfile to path + if not args['lpfile'] == False: + network.model.write(args['lpfile'], io_options={'symbolic_solver_labels': + True}) + # write PyPSA results to csv to path + if not args['results'] == False: + results_to_csv(network, args['results']) + + return network + +# execute etrago function +network = etrago(args) + +# plots + +# make a line loading plot +plot_line_loading(network) + +# plot stacked sum of nominal power for each generator type and timestep +plot_stacked_gen(network, resolution="MW") + +# plot to show extendable storages +storage_distribution(network) From a46621a10ac44129149f460df96c3ce5d1513d1c Mon Sep 17 00:00:00 2001 From: ulf Date: Wed, 23 Aug 2017 12:09:15 +0200 Subject: [PATCH 27/65] adjust transformer x --- etrago/appl.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/etrago/appl.py b/etrago/appl.py index 514f216c..c4e97952 100644 --- a/etrago/appl.py +++ b/etrago/appl.py @@ -24,18 +24,18 @@ args = {'network_clustering':False, 'db': 'oedb', # db session - 'gridversion':None, #None for model_draft or Version number (e.g. v0.2.10) for grid schema + 'gridversion':'v0.2.11', #None for model_draft or Version number (e.g. v0.2.10) for grid schema 'method': 'lopf', # lopf or pf - 'pf_post_lopf': False, #state whether you want to perform a pf after a lopf simulation - 'start_h': 2323, - 'end_h' : 2324, + 'pf_post_lopf': True, #state whether you want to perform a pf after a lopf simulation + 'start_h': 2320, + 'end_h' : 2325, 'scn_name': 'SH Status Quo', - 'ormcls_prefix': 'EgoGridPfHv', #if gridversion:'version-number' then 'EgoPfHv', if gridversion:None then 'EgoGridPfHv' + 'ormcls_prefix': 'EgoPfHv', #if gridversion:'version-number' then 'EgoPfHv', if gridversion:None then 'EgoGridPfHv' 'lpfile': False, # state if and where you want to save pyomo's lp file: False or '/path/tofolder' 'results': False , # state if and where you want to save results as csv: False or '/path/tofolder' 'solver': 'gurobi', #glpk, cplex or gurobi 'branch_capacity_factor': 1, #to globally extend or lower branch capacities - 'storage_extendable':True, + 'storage_extendable':False, 'load_shedding':True, 'generator_noise':False, 'parallelisation':False} @@ -61,8 +61,9 @@ def etrago(args): noise_values = network.generators.marginal_cost + abs(np.random.normal(0,0.001,len(network.generators.marginal_cost))) np.savetxt("noise_values.csv", noise_values, delimiter=",") noise_values = genfromtxt('noise_values.csv', delimiter=',') - - network.transformers.x=network.transformers.x*0.01 + + # TEMPORAL: adjust x of transformers to a more realistic order of magnitude until bug is fixed in data processing (v0.3) + network.transformers.x=network.transformers.x*0.0001 if args['branch_capacity_factor']: network.lines.s_nom = network.lines.s_nom*args['branch_capacity_factor'] From 8b2a92455dd2ce3e94345a1cae73401e190da432 Mon Sep 17 00:00:00 2001 From: wolfbunke Date: Wed, 23 Aug 2017 13:21:59 +0200 Subject: [PATCH 28/65] add file.lp --- etrago/scenario_setting.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etrago/scenario_setting.json b/etrago/scenario_setting.json index 215a8a60..3fac869c 100644 --- a/etrago/scenario_setting.json +++ b/etrago/scenario_setting.json @@ -8,8 +8,8 @@ "end_h" : 2324, "scn_name": "SH Status Quo", "ormcls_prefix": "EgoPfHv", -"lpfile": false, -"results": false, +"lpfile": "/home/dozeumbuw/eTraGo/src/etrago/etrago/results/file.lp", +"results": "/home/dozeumbuw/eTraGo/src/etrago/etrago/results/", "solver": "gurobi", "branch_capacity_factor": 1, "storage_extendable": true, From 94e15c85e6b01088da7705cc146e1af984c3c043 Mon Sep 17 00:00:00 2001 From: ulfmueller Date: Wed, 23 Aug 2017 13:38:20 +0200 Subject: [PATCH 29/65] Update appl.py --- etrago/appl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etrago/appl.py b/etrago/appl.py index 91b96f72..749ccc4e 100644 --- a/etrago/appl.py +++ b/etrago/appl.py @@ -31,7 +31,7 @@ 'end_h' : 2324, 'scn_name': 'SH Status Quo', 'ormcls_prefix': 'EgoGridPfHv', #if gridversion:'version-number' then 'EgoPfHv', if gridversion:None then 'EgoGridPfHv' - 'lpfile': False, # state if and where you want to save pyomo's lp file: False or '/path/tofolder' + 'lpfile': False, # state if and where you want to save pyomo's lp file: False or '/path/tofolder/file.lp' 'results': False, # state if and where you want to save results as csv: False or '/path/tofolder' 'solver': 'gurobi', #glpk, cplex or gurobi 'branch_capacity_factor': 1, #to globally extend or lower branch capacities From eb4830805027b5ea62ec28a5a52096896a066ac4 Mon Sep 17 00:00:00 2001 From: Lukas Date: Wed, 23 Aug 2017 14:18:29 +0200 Subject: [PATCH 30/65] added function to calculate line losses --- etrago/appl.py | 7 ++++--- etrago/extras/utilities.py | 33 ++++++++++++++++++++++++++++++--- 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/etrago/appl.py b/etrago/appl.py index c4e97952..c47445b3 100644 --- a/etrago/appl.py +++ b/etrago/appl.py @@ -19,7 +19,7 @@ from egopowerflow.tools.plot import (plot_line_loading, plot_stacked_gen, add_coordinates, curtailment, gen_dist, storage_distribution) -from etrago.extras.utilities import load_shedding, data_manipulation_sh, results_to_csv, parallelisation, pf_post_lopf +from etrago.extras.utilities import load_shedding, data_manipulation_sh, results_to_csv, parallelisation, pf_post_lopf, calc_line_losses from etrago.cluster.networkclustering import busmap_from_psql, cluster_on_extra_high_voltage args = {'network_clustering':False, @@ -32,7 +32,7 @@ 'scn_name': 'SH Status Quo', 'ormcls_prefix': 'EgoPfHv', #if gridversion:'version-number' then 'EgoPfHv', if gridversion:None then 'EgoGridPfHv' 'lpfile': False, # state if and where you want to save pyomo's lp file: False or '/path/tofolder' - 'results': False , # state if and where you want to save results as csv: False or '/path/tofolder' + 'results': '/srv/ES2050/open_eGo/AP3/lopf_results/gurobi_sh_pf_1' , # state if and where you want to save results as csv: False or '/path/tofolder' 'solver': 'gurobi', #glpk, cplex or gurobi 'branch_capacity_factor': 1, #to globally extend or lower branch capacities 'storage_extendable':False, @@ -108,8 +108,10 @@ def etrago(args): # start non-linear powerflow simulation elif args['method'] == 'pf': network.pf(scenario.timeindex) + if args['pf_post_lopf']: pf_post_lopf(network, scenario) + calc_line_losses(network) # write lpfile to path if not args['lpfile'] == False: @@ -136,6 +138,5 @@ def etrago(args): # plot to show extendable storages storage_distribution(network) - # close session #session.close() diff --git a/etrago/extras/utilities.py b/etrago/extras/utilities.py index f7af6f51..dd8378c1 100644 --- a/etrago/extras/utilities.py +++ b/etrago/extras/utilities.py @@ -192,8 +192,35 @@ def pf_post_lopf(network, scenario): #execute non-linear pf network_pf.pf(scenario.timeindex, use_seed=True) - #calculate p line losses - #todo - return network_pf +def calc_line_losses(network): + """ Calculate losses per line with PF result data + ---------- + network : :class:`pypsa.Network + Overall container of PyPSA + s0 : series + apparent power of line + i0 : series + current of line + ------- + + """ + + # calculate apparent power S = sqrt(p² + q²) + s0 = ((network.lines_t.p0**2 + network.lines_t.q0**2).\ + apply(np.sqrt)) + # calculate current I = S / U + i0 = s0 / network.lines.v_nom + # calculate losses per line and timestep network.lines_t.line_losses = I² * R + p_loss_t = i0**2 * network.lines.r + network.lines_t.line_losses = p_loss_t + # calculate total losses per line + p_loss = np.sum(p_loss_t) + # calculate total losses on all lines + p_loss_overall = sum(p_loss) + print("Total lines losses for all snapshots and lines [MW]:",p_loss_overall) + + return + + From 3610b2b4f431baf9b1bdbd37cf4ac974f0d483b9 Mon Sep 17 00:00:00 2001 From: wolfbunke Date: Wed, 23 Aug 2017 14:21:05 +0200 Subject: [PATCH 31/65] change import --- etrago/k_means_testing.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/etrago/k_means_testing.py b/etrago/k_means_testing.py index e75f591b..702c4747 100644 --- a/etrago/k_means_testing.py +++ b/etrago/k_means_testing.py @@ -23,7 +23,6 @@ """ - __copyright__ = "tba" __license__ = "tba" __author__ = "tba" @@ -40,13 +39,8 @@ storage_distribution) from etrago.extras.utilities import load_shedding, data_manipulation_sh, results_to_csv, parallelisation, pf_post_lopf from etrago.cluster.networkclustering import busmap_from_psql, cluster_on_extra_high_voltage - -# imports for k-mean -from pypsa.networkclustering import busmap_by_kmeans, get_clustering_from_busmap -from cluster.networkclustering import busmap_from_psql, cluster_on_extra_high_voltage from pypsa.networkclustering import busmap_by_kmeans, get_clustering_from_busmap import pandas as pd -import numpy as np import json # import scenario settings **args From 40f6ea26e62c7cb4ff894c0ff799f6ce7e0f85e5 Mon Sep 17 00:00:00 2001 From: Lukas Date: Wed, 23 Aug 2017 14:51:00 +0200 Subject: [PATCH 32/65] quick fix of calc_grid_losses --- etrago/extras/utilities.py | 1 + 1 file changed, 1 insertion(+) diff --git a/etrago/extras/utilities.py b/etrago/extras/utilities.py index dd8378c1..8e21f142 100644 --- a/etrago/extras/utilities.py +++ b/etrago/extras/utilities.py @@ -1,4 +1,5 @@ import pandas as pd +import numpy as np import os import time From 6ec1ef6e0cc4534953e8f5a3cd2faea2aacd767e Mon Sep 17 00:00:00 2001 From: mariusves Date: Mon, 28 Aug 2017 11:10:39 +0200 Subject: [PATCH 33/65] fix generator id for load shedding --- etrago/extras/utilities.py | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/etrago/extras/utilities.py b/etrago/extras/utilities.py index 59021cd0..b017e879 100644 --- a/etrago/extras/utilities.py +++ b/etrago/extras/utilities.py @@ -108,15 +108,20 @@ def load_shedding (network, **kwargs): marginal_cost = kwargs.get('marginal_cost', marginal_cost_def) p_nom = kwargs.get('p_nom', p_nom_def) - + network.add("Carrier", "load") + start = network.buses.index.astype(int).max() + nums = len(network.buses.index) + end = start+nums + index = list(range(start,end)) + index = [str(x) for x in index] network.import_components_from_dataframe( pd.DataFrame( dict(marginal_cost=marginal_cost, p_nom=p_nom, carrier='load shedding', bus=network.buses.index), - index=network.buses.index + '00000'), + index=index), "Generator" ) return @@ -134,8 +139,8 @@ def data_manipulation_sh (network): network.add("Bus", new_bus,carrier='AC', v_nom=220, x=10.760835, y=53.909745) network.add("Transformer", new_trafo, bus0="25536", bus1=new_bus, x=1.29960, tap_ratio=1, s_nom=1600) network.add("Line",new_line, bus0="26387",bus1=new_bus, x=0.0001, s_nom=1600) - network.lines.cables[new_line]=3.0 - + network.lines.loc[new_line,'cables']=3.0 + #bus geom point_bus1 = Point(10.760835,53.909745) network.buses.geom[new_bus]= from_shape(point_bus1, 4326) @@ -148,6 +153,18 @@ def data_manipulation_sh (network): network.transformers.geom[new_trafo] = from_shape(MultiLineString([LineString([to_shape(network.buses.geom['25536']),point_bus1])]),4326) network.transformers.topo[new_trafo] = from_shape(LineString([to_shape(network.buses.geom['25536']),point_bus1]),4326) +# #bus geom +# point_bus1 = Point(10.760835,53.909745) +# network.buses.loc[new_bus,'geom'] = from_shape(point_bus1, 4326) +# +# #line geom/topo +# network.lines.loc[new_line,'geom'] = from_shape(MultiLineString([LineString([to_shape(network.buses.geom['26387']),point_bus1])]),4326) +# network.lines.loc[new_line,'topo'] = from_shape(LineString([to_shape(network.buses.geom['26387']),point_bus1]),4326) +# +# #trafo geom/topo +# network.transformers.loc[new_trafo,'geom'] = from_shape(MultiLineString([LineString([to_shape(network.buses.geom['25536']),point_bus1])]),4326) +# network.transformers.loc[new_trafo,'topo'] = from_shape(LineString([to_shape(network.buses.geom['25536']),point_bus1]),4326) + return def results_to_csv(network, path): From 5a4fc1eb47c138f8b7bbfd745dfe1e6254be6315 Mon Sep 17 00:00:00 2001 From: wolfbunke Date: Tue, 29 Aug 2017 09:22:12 +0200 Subject: [PATCH 34/65] add k-mean to etrago() --- etrago/k_means_testing.py | 68 +++++++++++++++++++++------------------ 1 file changed, 37 insertions(+), 31 deletions(-) diff --git a/etrago/k_means_testing.py b/etrago/k_means_testing.py index 702c4747..02a5610f 100644 --- a/etrago/k_means_testing.py +++ b/etrago/k_means_testing.py @@ -104,6 +104,43 @@ def etrago(args): busmap = busmap_from_psql(network, session, scn_name=args['scn_name']) network = cluster_on_extra_high_voltage(network, busmap, with_time=True) + print('start k-mean clustering') + # prepare k-mean + # k-means clustering (first try) + network.generators.control="PV" + network.buses['v_nom'] = 380. + # problem our lines have no v_nom. this is implicitly defined by the connected buses: + network.lines["v_nom"] = network.lines.bus0.map(network.buses.v_nom) + + # adjust the x of the lines which are not 380. + lines_v_nom_b = network.lines.v_nom != 380 + network.lines.loc[lines_v_nom_b, 'x'] *= (380./network.lines.loc[lines_v_nom_b, 'v_nom'])**2 + network.lines.loc[lines_v_nom_b, 'v_nom'] = 380. + + trafo_index = network.transformers.index + + network.import_components_from_dataframe( + network.transformers.loc[:,['bus0','bus1','x','s_nom']] + .assign(x=0.1*380**2/2000) + .set_index('T' + trafo_index), + 'Line') + network.transformers.drop(trafo_index, inplace=True) + + for attr in network.transformers_t: + network.transformers_t[attr] = network.transformers_t[attr].reindex(columns=[]) + + + # k-mean clustering + busmap = busmap_by_kmeans(network, bus_weightings=pd.Series(np.repeat(1, + len(network.buses)), index=network.buses.index) , n_clusters= 50) + # ToDo: + #change np.repeat(1, len(network.buses) to load and conv P_max + + clustering = get_clustering_from_busmap(network, busmap) + network = clustering.network + #network = cluster_on_extra_high_voltage(network, busmap, with_time=True) + + # parallisation if args['parallelisation']: parallelisation(network, start_h=args['start_h'], end_h=args['end_h'],group_size=1, solver_name=args['solver']) @@ -133,36 +170,6 @@ def etrago(args): network = etrago(args) -network.generators.control="PV" - -network.buses['v_nom'] = 380. - -# TODO adjust the x of the lines which are not 380. problem our lines have no v_nom. this is implicitly defined by the connected buses. Generally it should look something like the following: -#lines_v_nom_b = network.lines.v_nom != 380 -#network.lines.loc[lines_v_nom_b, 'x'] *= (380./network.lines.loc[lines_v_nom_b, 'v_nom'])**2 -#network.lines.loc[lines_v_nom_b, 'v_nom'] = 380. - - -trafo_index = network.transformers.index - -network.import_components_from_dataframe( - network.transformers.loc[:,['bus0','bus1','x','s_nom']] - .assign(x=0.1*380**2/2000) - .set_index('T' + trafo_index), - 'Line' -) - -network.transformers.drop(trafo_index, inplace=True) -for attr in network.transformers_t: - network.transformers_t[attr] = network.transformers_t[attr].reindex(columns=[]) - -busmap = busmap_by_kmeans(network, bus_weightings=pd.Series(np.repeat(1, len(network.buses)), index=network.buses.index) , n_clusters= 50) - - -clustering = get_clustering_from_busmap(network, busmap) -network = clustering.network -#network = cluster_on_extra_high_voltage(network, busmap, with_time=True) - # plots # make a line loading plot @@ -171,7 +178,6 @@ def etrago(args): #plot stacked sum of nominal power for each generator type and timestep #plot_stacked_gen(network, resolution="MW") - # close session #session.close() From 0b7305cf1e43142ad9905e11e09da070695561bc Mon Sep 17 00:00:00 2001 From: wolfbunke Date: Tue, 29 Aug 2017 11:00:44 +0200 Subject: [PATCH 35/65] add var k_mean_clustering to args --- etrago/k_means_testing.py | 53 +++++++++++++++++++----------------- etrago/scenario_setting.json | 3 +- 2 files changed, 30 insertions(+), 26 deletions(-) diff --git a/etrago/k_means_testing.py b/etrago/k_means_testing.py index 02a5610f..ebb0d19c 100644 --- a/etrago/k_means_testing.py +++ b/etrago/k_means_testing.py @@ -104,41 +104,44 @@ def etrago(args): busmap = busmap_from_psql(network, session, scn_name=args['scn_name']) network = cluster_on_extra_high_voltage(network, busmap, with_time=True) - print('start k-mean clustering') - # prepare k-mean - # k-means clustering (first try) - network.generators.control="PV" - network.buses['v_nom'] = 380. - # problem our lines have no v_nom. this is implicitly defined by the connected buses: - network.lines["v_nom"] = network.lines.bus0.map(network.buses.v_nom) + + + if args['k_mean_clustering']: + print('start k-mean clustering') + # prepare k-mean + # k-means clustering (first try) + network.generators.control="PV" + network.buses['v_nom'] = 380. + # problem our lines have no v_nom. this is implicitly defined by the connected buses: + network.lines["v_nom"] = network.lines.bus0.map(network.buses.v_nom) - # adjust the x of the lines which are not 380. - lines_v_nom_b = network.lines.v_nom != 380 - network.lines.loc[lines_v_nom_b, 'x'] *= (380./network.lines.loc[lines_v_nom_b, 'v_nom'])**2 - network.lines.loc[lines_v_nom_b, 'v_nom'] = 380. + # adjust the x of the lines which are not 380. + lines_v_nom_b = network.lines.v_nom != 380 + network.lines.loc[lines_v_nom_b, 'x'] *= (380./network.lines.loc[lines_v_nom_b, 'v_nom'])**2 + network.lines.loc[lines_v_nom_b, 'v_nom'] = 380. - trafo_index = network.transformers.index + trafo_index = network.transformers.index - network.import_components_from_dataframe( - network.transformers.loc[:,['bus0','bus1','x','s_nom']] - .assign(x=0.1*380**2/2000) - .set_index('T' + trafo_index), - 'Line') - network.transformers.drop(trafo_index, inplace=True) + network.import_components_from_dataframe( + network.transformers.loc[:,['bus0','bus1','x','s_nom']] + .assign(x=0.1*380**2/2000) + .set_index('T' + trafo_index), + 'Line') + network.transformers.drop(trafo_index, inplace=True) - for attr in network.transformers_t: - network.transformers_t[attr] = network.transformers_t[attr].reindex(columns=[]) + for attr in network.transformers_t: + network.transformers_t[attr] = network.transformers_t[attr].reindex(columns=[]) - # k-mean clustering - busmap = busmap_by_kmeans(network, bus_weightings=pd.Series(np.repeat(1, + # k-mean clustering + busmap = busmap_by_kmeans(network, bus_weightings=pd.Series(np.repeat(1, len(network.buses)), index=network.buses.index) , n_clusters= 50) # ToDo: #change np.repeat(1, len(network.buses) to load and conv P_max - clustering = get_clustering_from_busmap(network, busmap) - network = clustering.network - #network = cluster_on_extra_high_voltage(network, busmap, with_time=True) + clustering = get_clustering_from_busmap(network, busmap) + network = clustering.network + #network = cluster_on_extra_high_voltage(network, busmap, with_time=True) # parallisation diff --git a/etrago/scenario_setting.json b/etrago/scenario_setting.json index 3fac869c..b4a9f242 100644 --- a/etrago/scenario_setting.json +++ b/etrago/scenario_setting.json @@ -15,6 +15,7 @@ "storage_extendable": true, "load_shedding": true, "generator_noise": true, -"parallelisation": false +"parallelisation": false, +"k_mean_clustering": true } From a86395191bb2fdea81756b9d22ce74b86611f4a8 Mon Sep 17 00:00:00 2001 From: MariusVes Date: Tue, 29 Aug 2017 17:27:26 +0200 Subject: [PATCH 36/65] fix small merge error --- etrago/extras/utilities.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/etrago/extras/utilities.py b/etrago/extras/utilities.py index 69522186..afa6ffc0 100644 --- a/etrago/extras/utilities.py +++ b/etrago/extras/utilities.py @@ -132,10 +132,6 @@ def data_manipulation_sh (network): from shapely.geometry import Point, LineString, MultiLineString from geoalchemy2.shape import from_shape, to_shape - new_bus = str(network.buses.index.astype(int).max()+1) - new_trafo = str(network.transformers.index.astype(int).max()+1) - new_line = str(network.lines.index.astype(int).max()+1) - #add connection from Luebeck to Siems new_bus = str(int(network.buses.index.max())+1) From 02427635f1fce13cd8007b0c705f785d1b1df43c Mon Sep 17 00:00:00 2001 From: wolfbunke Date: Wed, 30 Aug 2017 09:18:40 +0200 Subject: [PATCH 37/65] update k-mean wip --- etrago/k_means_testing.py | 262 +++++++++++++++++++++----------------- 1 file changed, 145 insertions(+), 117 deletions(-) diff --git a/etrago/k_means_testing.py b/etrago/k_means_testing.py index ebb0d19c..4557f91c 100644 --- a/etrago/k_means_testing.py +++ b/etrago/k_means_testing.py @@ -43,136 +43,164 @@ import pandas as pd import json + # import scenario settings **args with open('scenario_setting.json') as f: scenario_setting = json.load(f) args = scenario_setting -def etrago(args): - session = oedb_session(args['db']) - - # additional arguments cfgpath, version, prefix - scenario = NetworkScenario(session, - version=args['gridversion'], - prefix=args['ormcls_prefix'], - method=args['method'], - start_h=args['start_h'], - end_h=args['end_h'], - scn_name=args['scn_name']) - - network = scenario.build_network() - - # add coordinates - network = add_coordinates(network) - - # TEMPORARY vague adjustment due to transformer bug in data processing - #network.transformers.x=network.transformers.x*0.01 - - - if args['branch_capacity_factor']: - network.lines.s_nom = network.lines.s_nom*args['branch_capacity_factor'] - network.transformers.s_nom = network.transformers.s_nom*args['branch_capacity_factor'] - - if args['generator_noise']: - # create generator noise - noise_values = network.generators.marginal_cost + abs(np.random.normal(0,0.001,len(network.generators.marginal_cost))) - np.savetxt("noise_values.csv", noise_values, delimiter=",") - noise_values = genfromtxt('noise_values.csv', delimiter=',') - # add random noise to all generator - network.generators.marginal_cost = noise_values - - if args['storage_extendable']: - # set virtual storages to be extendable - if network.storage_units.source.any()=='extendable_storage': - network.storage_units.p_nom_extendable = True - # set virtual storage costs with regards to snapshot length - network.storage_units.capital_cost = (network.storage_units.capital_cost / - (8760//(args['end_h']-args['start_h']+1))) - - # for SH scenario run do data preperation: - if args['scn_name'] == 'SH Status Quo': - data_manipulation_sh(network) - - #load shedding in order to hunt infeasibilities - if args['load_shedding']: - load_shedding(network) - - # network clustering - if args['network_clustering']: - network.generators.control="PV" - busmap = busmap_from_psql(network, session, scn_name=args['scn_name']) - network = cluster_on_extra_high_voltage(network, busmap, with_time=True) - - - - if args['k_mean_clustering']: - print('start k-mean clustering') - # prepare k-mean - # k-means clustering (first try) - network.generators.control="PV" - network.buses['v_nom'] = 380. - # problem our lines have no v_nom. this is implicitly defined by the connected buses: - network.lines["v_nom"] = network.lines.bus0.map(network.buses.v_nom) - - # adjust the x of the lines which are not 380. - lines_v_nom_b = network.lines.v_nom != 380 - network.lines.loc[lines_v_nom_b, 'x'] *= (380./network.lines.loc[lines_v_nom_b, 'v_nom'])**2 - network.lines.loc[lines_v_nom_b, 'v_nom'] = 380. - - trafo_index = network.transformers.index - - network.import_components_from_dataframe( - network.transformers.loc[:,['bus0','bus1','x','s_nom']] - .assign(x=0.1*380**2/2000) - .set_index('T' + trafo_index), - 'Line') - network.transformers.drop(trafo_index, inplace=True) - - for attr in network.transformers_t: - network.transformers_t[attr] = network.transformers_t[attr].reindex(columns=[]) - - - # k-mean clustering - busmap = busmap_by_kmeans(network, bus_weightings=pd.Series(np.repeat(1, - len(network.buses)), index=network.buses.index) , n_clusters= 50) - # ToDo: - #change np.repeat(1, len(network.buses) to load and conv P_max - - clustering = get_clustering_from_busmap(network, busmap) - network = clustering.network - #network = cluster_on_extra_high_voltage(network, busmap, with_time=True) + +# function for k-mean ToDo: move to cluster.networkclustering or extras.utilities +def weighting_for_scenario(x): + b_i = x.index + g = normed(gen.reindex(b_i, fill_value=0)) + l = normed(load.reindex(b_i, fill_value=0)) + w= g + l + return (w * (100. / w.max())).astype(int) + +def normed(x): + return (x/x.sum()).fillna(0.) + - # parallisation - if args['parallelisation']: - parallelisation(network, start_h=args['start_h'], end_h=args['end_h'],group_size=1, solver_name=args['solver']) - # start linear optimal powerflow calculations - elif args['method'] == 'lopf': - x = time.time() - network.lopf(scenario.timeindex, solver_name=args['solver']) - y = time.time() - z = (y - x) / 60 # z is time for lopf in minutes - # start non-linear powerflow simulation - elif args['method'] == 'pf': - network.pf(scenario.timeindex) - if args['pf_post_lopf']: - pf_post_lopf(network, scenario) - - # write lpfile to path - if not args['lpfile'] == False: - network.model.write(args['lpfile'], io_options={'symbolic_solver_labels': - True}) - # write PyPSA results to csv to path - if not args['results'] == False: - results_to_csv(network, args['results']) - - return network + +def etrago(args): + session = oedb_session(args['db']) + + # additional arguments cfgpath, version, prefix + scenario = NetworkScenario(session, + version=args['gridversion'], + prefix=args['ormcls_prefix'], + method=args['method'], + start_h=args['start_h'], + end_h=args['end_h'], + scn_name=args['scn_name']) + + network = scenario.build_network() + + # add coordinates + network = add_coordinates(network) + + # TEMPORARY vague adjustment due to transformer bug in data processing + #network.transformers.x=network.transformers.x*0.01 + + + if args['branch_capacity_factor']: + network.lines.s_nom = network.lines.s_nom*args['branch_capacity_factor'] + network.transformers.s_nom = network.transformers.s_nom*args['branch_capacity_factor'] + + if args['generator_noise']: + # create generator noise + noise_values = network.generators.marginal_cost + abs(np.random.normal(0,0.001,len(network.generators.marginal_cost))) + np.savetxt("noise_values.csv", noise_values, delimiter=",") + noise_values = genfromtxt('noise_values.csv', delimiter=',') + # add random noise to all generator + network.generators.marginal_cost = noise_values + + if args['storage_extendable']: + # set virtual storages to be extendable + if network.storage_units.source.any()=='extendable_storage': + network.storage_units.p_nom_extendable = True + # set virtual storage costs with regards to snapshot length + network.storage_units.capital_cost = (network.storage_units.capital_cost / + (8760//(args['end_h']-args['start_h']+1))) + + # for SH scenario run do data preperation: + if args['scn_name'] == 'SH Status Quo': + data_manipulation_sh(network) + + #load shedding in order to hunt infeasibilities + if args['load_shedding']: + load_shedding(network) + + # network clustering + if args['network_clustering']: + network.generators.control="PV" + busmap = busmap_from_psql(network, session, scn_name=args['scn_name']) + network = cluster_on_extra_high_voltage(network, busmap, with_time=True) + + + + if args['k_mean_clustering']: + print('start k-mean clustering') + # prepare k-mean + # k-means clustering (first try) + network.generators.control="PV" + network.buses['v_nom'] = 380. + # problem our lines have no v_nom. this is implicitly defined by the connected buses: + network.lines["v_nom"] = network.lines.bus0.map(network.buses.v_nom) + + # adjust the x of the lines which are not 380. + lines_v_nom_b = network.lines.v_nom != 380 + network.lines.loc[lines_v_nom_b, 'x'] *= (380./network.lines.loc[lines_v_nom_b, 'v_nom'])**2 + network.lines.loc[lines_v_nom_b, 'v_nom'] = 380. + + trafo_index = network.transformers.index + + network.import_components_from_dataframe( + network.transformers.loc[:,['bus0','bus1','x','s_nom']] + .assign(x=0.1*380**2/2000) + .set_index('T' + trafo_index), + 'Line') + network.transformers.drop(trafo_index, inplace=True) + + for attr in network.transformers_t: + network.transformers_t[attr] = network.transformers_t[attr].reindex(columns=[]) + + # ToDo: + #change np.repeat(1, len(network.buses) to load and conv P_max + conv_types = {'biomass', 'run_of_river', 'gas', 'oil','coal', 'waste','uranium'} + # Attention: network.generators.carrier.unique() + # conv_types only for SH scenario defined! + gen = (network.generators.loc[network.generators.carrier.isin(conv_types) + ].groupby('bus').p_nom.sum().reindex(network.buses.index, + fill_value=0.) + network.storage_units.loc[network.storage_units.carrier.isin(conv_types) + ].groupby('bus').p_nom.sum().reindex(network.buses.index, fill_value=0.)) + + load = network.loads_t.p_set.mean().groupby(network.loads.bus).sum() + + # k-mean clustering + busmap = busmap_by_kmeans(network, bus_weightings=pd.Series(np.repeat(1, + len(network.buses)), index=network.buses.index) , n_clusters= 10) + + # ToDo change function in order to use bus_strategies or similar + clustering = get_clustering_from_busmap(network, busmap) + network = clustering.network + #network = cluster_on_extra_high_voltage(network, busmap, with_time=True) + + + # parallisation + if args['parallelisation']: + parallelisation(network, start_h=args['start_h'], end_h=args['end_h'],group_size=1, solver_name=args['solver']) + # start linear optimal powerflow calculations + elif args['method'] == 'lopf': + x = time.time() + network.lopf(scenario.timeindex, solver_name=args['solver']) + y = time.time() + z = (y - x) / 60 # z is time for lopf in minutes + # start non-linear powerflow simulation + elif args['method'] == 'pf': + network.pf(scenario.timeindex) + if args['pf_post_lopf']: + pf_post_lopf(network, scenario) + + # write lpfile to path + if not args['lpfile'] == False: + network.model.write(args['lpfile'], io_options={'symbolic_solver_labels': + True}) + # write PyPSA results to csv to path + if not args['results'] == False: + results_to_csv(network, args['results']) + + return network # execute etrago function network = etrago(args) +#network.plot(bus_sizes=(2*weighting_for_scenario(network.buses.query('scn_name=="SH Status Quo"'))).reindex(network.buses.index, fill_value=1)) + # plots # make a line loading plot From c29820a99c27937c512c29846492f53dba0aa649 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 31 Aug 2017 14:04:32 +0200 Subject: [PATCH 38/65] Changes Extra_functionality --- etrago/line_loading_minimization_appl.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/etrago/line_loading_minimization_appl.py b/etrago/line_loading_minimization_appl.py index cacb7f5c..9643d64a 100644 --- a/etrago/line_loading_minimization_appl.py +++ b/etrago/line_loading_minimization_appl.py @@ -89,11 +89,11 @@ def extra_functionality(network,snapshots): network.model.number2 = Var(network.model.passive_branch_p_index, within = PositiveReals) def cRule(model, c, l, t): - return (model.number1[c, l, t] + model.number2[c, l, t] == model.passive_branch_p[c, l, t]) + return (model.number1[c, l, t] - model.number2[c, l, t] == model.passive_branch_p[c, l, t]) network.model.cRule=Constraint(network.model.passive_branch_p_index, rule=cRule) - network.model.objective.expr += 0.01* sum(network.model.passive_branch_p[i] for i in network.model.passive_branch_p_index) + network.model.objective.expr += 0.01* sum(network.model.number1[i] + network.model.number2[i] for i in network.model.passive_branch_p_index) # start powerflow calculations x = time.time() @@ -105,7 +105,7 @@ def cRule(model, c, l, t): -# write results +#write results network.model.write(args['outfile'], io_options={'symbolic_solver_labels':True}) results_to_csv(network, args['results']) From 2bf6ecea7dfb1a49571a918b63814a9019b67041 Mon Sep 17 00:00:00 2001 From: ulf Date: Fri, 1 Sep 2017 12:03:39 +0200 Subject: [PATCH 39/65] adjust epsilon --- etrago/line_loading_minimization_appl.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/etrago/line_loading_minimization_appl.py b/etrago/line_loading_minimization_appl.py index 9643d64a..ae1eb9c2 100644 --- a/etrago/line_loading_minimization_appl.py +++ b/etrago/line_loading_minimization_appl.py @@ -22,17 +22,17 @@ args = {'network_clustering':False, 'db': 'oedb', # db session - 'gridversion':None, #None for model_draft or Version number (e.g. v0.2.10) for grid schema + 'gridversion':'v0.2.11', #None for model_draft or Version number (e.g. v0.2.10) for grid schema 'method': 'lopf', # lopf or pf - 'start_h': 2301, - 'end_h' : 2303, + 'start_h': 2320, + 'end_h' : 2321, 'scn_name': 'SH Status Quo', - 'ormcls_prefix': 'EgoGridPfHv', #if gridversion:'version-number' then 'EgoPfHv', if gridversion:None then 'EgoGridPfHv' - 'outfile': '/path', # state if and where you want to save pyomo's lp file + 'ormcls_prefix': 'EgoPfHv', #if gridversion:'version-number' then 'EgoPfHv', if gridversion:None then 'EgoGridPfHv' + 'outfile': '/path/file.lp', # state if and where you want to save pyomo's lp file 'results': '/path', # state if and where you want to save results as csv 'solver': 'gurobi', #glpk, cplex or gurobi 'branch_capacity_factor': 1, #to globally extend or lower branch capacities - 'storage_extendable':True, + 'storage_extendable':False, 'load_shedding':True, 'generator_noise':False} @@ -93,12 +93,11 @@ def cRule(model, c, l, t): network.model.cRule=Constraint(network.model.passive_branch_p_index, rule=cRule) - network.model.objective.expr += 0.01* sum(network.model.number1[i] + network.model.number2[i] for i in network.model.passive_branch_p_index) + network.model.objective.expr += 0.00001* sum(network.model.number1[i] + network.model.number2[i] for i in network.model.passive_branch_p_index) # start powerflow calculations x = time.time() -network.lopf(scenario.timeindex,solver_name=args['solver'], - extra_functionality=extra_functionality) +network.lopf(scenario.timeindex,solver_name=args['solver'], extra_functionality=extra_functionality) y = time.time() z = (y - x) / 60 # z is time for lopf in minutes @@ -108,7 +107,7 @@ def cRule(model, c, l, t): #write results network.model.write(args['outfile'], io_options={'symbolic_solver_labels':True}) -results_to_csv(network, args['results']) +#results_to_csv(network, args['results']) # plots From e95f770dcaa03bdd477b277707a4aaa770016b7d Mon Sep 17 00:00:00 2001 From: ulf Date: Tue, 5 Sep 2017 14:00:11 +0200 Subject: [PATCH 40/65] add appl.py --- etrago/appl.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 etrago/appl.py diff --git a/etrago/appl.py b/etrago/appl.py new file mode 100644 index 00000000..e69de29b From 3c32c42fd8b246a621ec726334eb2e36144aba85 Mon Sep 17 00:00:00 2001 From: ulf Date: Tue, 5 Sep 2017 15:19:24 +0200 Subject: [PATCH 41/65] include line loading minimization in appl.py --- etrago/appl.py | 21 ++++++++++++++------- etrago/extras/utilities.py | 18 ++++++++++++++++-- etrago/line_loading_minimization_appl.py | 7 +++++-- 3 files changed, 35 insertions(+), 11 deletions(-) diff --git a/etrago/appl.py b/etrago/appl.py index e73ac924..4b69168b 100644 --- a/etrago/appl.py +++ b/etrago/appl.py @@ -19,18 +19,18 @@ from egopowerflow.tools.plot import (plot_line_loading, plot_stacked_gen, add_coordinates, curtailment, gen_dist, storage_distribution) -from etrago.extras.utilities import load_shedding, data_manipulation_sh, results_to_csv, parallelisation, pf_post_lopf +from etrago.extras.utilities import load_shedding, data_manipulation_sh, results_to_csv, parallelisation, pf_post_lopf, loading_minimization from etrago.cluster.networkclustering import busmap_from_psql, cluster_on_extra_high_voltage args = {'network_clustering':False, 'db': 'oedb', # db session - 'gridversion': None, #None for model_draft or Version number (e.g. v0.2.10) for grid schema + 'gridversion': 'v0.2.11', #None for model_draft or Version number (e.g. v0.2.10) for grid schema 'method': 'lopf', # lopf or pf 'pf_post_lopf':False , #state whether you want to perform a pf after a lopf simulation - 'start_h': 2323, - 'end_h' : 2324, + 'start_h': 2320, + 'end_h' : 2321, 'scn_name': 'SH Status Quo', - 'ormcls_prefix': 'EgoGridPfHv', #if gridversion:'version-number' then 'EgoPfHv', if gridversion:None then 'EgoGridPfHv' + 'ormcls_prefix': 'EgoPfHv', #if gridversion:'version-number' then 'EgoPfHv', if gridversion:None then 'EgoGridPfHv' 'lpfile': False, # state if and where you want to save pyomo's lp file: False or '/path/tofolder/file.lp' 'results': False, # state if and where you want to save results as csv: False or '/path/tofolder' 'export': False, # state if you want to export the results back to the database @@ -39,6 +39,7 @@ 'storage_extendable':True, 'load_shedding':True, 'generator_noise':True, + 'minimize_loading':False, 'parallelisation':False, 'comments': None} @@ -97,13 +98,19 @@ def etrago(args): busmap = busmap_from_psql(network, session, scn_name=args['scn_name']) network = cluster_on_extra_high_voltage(network, busmap, with_time=True) + # Branch loading minimization + if args['minimize_loading']: + extra_functionality = loading_minimization + else: + extra_functionality=None + # parallisation if args['parallelisation']: - parallelisation(network, start_h=args['start_h'], end_h=args['end_h'],group_size=1, solver_name=args['solver']) + parallelisation(network, start_h=args['start_h'], end_h=args['end_h'],group_size=1, solver_name=args['solver'], extra_functionality=extra_functionality) # start linear optimal powerflow calculations elif args['method'] == 'lopf': x = time.time() - network.lopf(scenario.timeindex, solver_name=args['solver']) + network.lopf(scenario.timeindex, solver_name=args['solver'], extra_functionality=extra_functionality) y = time.time() z = (y - x) / 60 # z is time for lopf in minutes # start non-linear powerflow simulation diff --git a/etrago/extras/utilities.py b/etrago/extras/utilities.py index afa6ffc0..550d252d 100644 --- a/etrago/extras/utilities.py +++ b/etrago/extras/utilities.py @@ -1,6 +1,7 @@ import pandas as pd import os import time +from pyomo.environ import (Var,Constraint, PositiveReals,ConcreteModel) def buses_of_vlvl(network, voltage_level): """ Get bus-ids of given voltage level(s). @@ -193,12 +194,12 @@ def results_to_csv(network, path): return -def parallelisation(network, start_h, end_h, group_size, solver_name): +def parallelisation(network, start_h, end_h, group_size, solver_name, extra_functionality=None): print("Performing linear OPF, {} snapshot(s) at a time:".format(group_size)) x = time.time() for i in range(int((end_h-start_h+1)/group_size)): - network.lopf(network.snapshots[group_size*i:group_size*i+group_size], solver_name=solver_name) + network.lopf(network.snapshots[group_size*i:group_size*i+group_size], solver_name=solver_name, extra_functionality=extra_functionality) y = time.time() @@ -234,3 +235,16 @@ def pf_post_lopf(network, scenario): return network_pf +def loading_minimization(network,snapshots): + + network.model.number1 = Var(network.model.passive_branch_p_index, within = PositiveReals) + network.model.number2 = Var(network.model.passive_branch_p_index, within = PositiveReals) + + def cRule(model, c, l, t): + return (model.number1[c, l, t] - model.number2[c, l, t] == model.passive_branch_p[c, l, t]) + + network.model.cRule=Constraint(network.model.passive_branch_p_index, rule=cRule) + + network.model.objective.expr += 0.00001* sum(network.model.number1[i] + network.model.number2[i] for i in network.model.passive_branch_p_index) + + diff --git a/etrago/line_loading_minimization_appl.py b/etrago/line_loading_minimization_appl.py index ae1eb9c2..33186da0 100644 --- a/etrago/line_loading_minimization_appl.py +++ b/etrago/line_loading_minimization_appl.py @@ -28,7 +28,7 @@ 'end_h' : 2321, 'scn_name': 'SH Status Quo', 'ormcls_prefix': 'EgoPfHv', #if gridversion:'version-number' then 'EgoPfHv', if gridversion:None then 'EgoGridPfHv' - 'outfile': '/path/file.lp', # state if and where you want to save pyomo's lp file + 'outfile': '/home/ulf/file.lp', # state if and where you want to save pyomo's lp file 'results': '/path', # state if and where you want to save results as csv 'solver': 'gurobi', #glpk, cplex or gurobi 'branch_capacity_factor': 1, #to globally extend or lower branch capacities @@ -83,7 +83,7 @@ busmap = busmap_from_psql(network, session, scn_name=args['scn_name']) network = cluster_on_extra_high_voltage(network, busmap, with_time=True) -def extra_functionality(network,snapshots): +def loading_minimization(network,snapshots): network.model.number1 = Var(network.model.passive_branch_p_index, within = PositiveReals) network.model.number2 = Var(network.model.passive_branch_p_index, within = PositiveReals) @@ -95,6 +95,9 @@ def cRule(model, c, l, t): network.model.objective.expr += 0.00001* sum(network.model.number1[i] + network.model.number2[i] for i in network.model.passive_branch_p_index) +#extra_functionality = loading_minimization +extra_functionality=None + # start powerflow calculations x = time.time() network.lopf(scenario.timeindex,solver_name=args['solver'], extra_functionality=extra_functionality) From ac7bb6b4011990cd36d94a1e0467d952333f16d2 Mon Sep 17 00:00:00 2001 From: ulf Date: Tue, 5 Sep 2017 15:21:28 +0200 Subject: [PATCH 42/65] deleted testing app --- etrago/line_loading_minimization_appl.py | 127 ----------------------- 1 file changed, 127 deletions(-) delete mode 100644 etrago/line_loading_minimization_appl.py diff --git a/etrago/line_loading_minimization_appl.py b/etrago/line_loading_minimization_appl.py deleted file mode 100644 index 33186da0..00000000 --- a/etrago/line_loading_minimization_appl.py +++ /dev/null @@ -1,127 +0,0 @@ -"""This is the docstring for the example.py module. Modules names should -have short, all-lowercase names. The module name may have underscores if -this improves readability. -Every module should have a docstring at the very top of the file. The -module's docstring may extend over multiple lines. If your docstring does -extend over multiple lines, the closing three quotation marks must be on -a line by itself, preferably preceded by a blank line.""" - -__copyright__ = "tba" -__license__ = "tba" -__author__ = "tba" - -import numpy as np -np.random.seed() -from egopowerflow.tools.tools import oedb_session -from egopowerflow.tools.io import NetworkScenario -from pyomo.environ import (Var,Constraint, PositiveReals,ConcreteModel) -import time -from egopowerflow.tools.plot import plot_line_loading, plot_stacked_gen, add_coordinates, curtailment, gen_dist, storage_distribution -from extras.utilities import load_shedding, data_manipulation_sh, results_to_csv -from cluster.networkclustering import busmap_from_psql, cluster_on_extra_high_voltage - -args = {'network_clustering':False, - 'db': 'oedb', # db session - 'gridversion':'v0.2.11', #None for model_draft or Version number (e.g. v0.2.10) for grid schema - 'method': 'lopf', # lopf or pf - 'start_h': 2320, - 'end_h' : 2321, - 'scn_name': 'SH Status Quo', - 'ormcls_prefix': 'EgoPfHv', #if gridversion:'version-number' then 'EgoPfHv', if gridversion:None then 'EgoGridPfHv' - 'outfile': '/home/ulf/file.lp', # state if and where you want to save pyomo's lp file - 'results': '/path', # state if and where you want to save results as csv - 'solver': 'gurobi', #glpk, cplex or gurobi - 'branch_capacity_factor': 1, #to globally extend or lower branch capacities - 'storage_extendable':False, - 'load_shedding':True, - 'generator_noise':False} - - -session = oedb_session(args['db']) - -# additional arguments cfgpath, version, prefix -scenario = NetworkScenario(session, - version=args['gridversion'], - prefix=args['ormcls_prefix'], - method=args['method'], - start_h=args['start_h'], - end_h=args['end_h'], - scn_name=args['scn_name']) - -network = scenario.build_network() - -# add coordinates -network = add_coordinates(network) - -if args['branch_capacity_factor']: - network.lines.s_nom = network.lines.s_nom*args['branch_capacity_factor'] - network.transformers.s_nom = network.transformers.s_nom*args['branch_capacity_factor'] - - -if args['generator_noise']: - # add random noise to all generators with marginal_cost of 0. - network.generators.marginal_cost[ network.generators.marginal_cost == 0] = abs(np.random.normal(0,0.00001,sum(network.generators.marginal_cost == 0))) - -if args['storage_extendable']: - # set virtual storages to be extendable - network.storage_units.p_nom_extendable = True - # set virtual storage costs with regards to snapshot length - network.storage_units.capital_cost = network.storage_units.capital_cost / (8760//(args['end_h']-args['start_h']+1)) - - -# for SH scenario run do data preperation: -if args['scn_name'] == 'SH Status Quo': - data_manipulation_sh(network) - -#load shedding in order to hunt infeasibilities -if args['load_shedding']: - load_shedding(network) - -# network clustering -if args['network_clustering']: - network.generators.control="PV" - busmap = busmap_from_psql(network, session, scn_name=args['scn_name']) - network = cluster_on_extra_high_voltage(network, busmap, with_time=True) - -def loading_minimization(network,snapshots): - - network.model.number1 = Var(network.model.passive_branch_p_index, within = PositiveReals) - network.model.number2 = Var(network.model.passive_branch_p_index, within = PositiveReals) - - def cRule(model, c, l, t): - return (model.number1[c, l, t] - model.number2[c, l, t] == model.passive_branch_p[c, l, t]) - - network.model.cRule=Constraint(network.model.passive_branch_p_index, rule=cRule) - - network.model.objective.expr += 0.00001* sum(network.model.number1[i] + network.model.number2[i] for i in network.model.passive_branch_p_index) - -#extra_functionality = loading_minimization -extra_functionality=None - -# start powerflow calculations -x = time.time() -network.lopf(scenario.timeindex,solver_name=args['solver'], extra_functionality=extra_functionality) -y = time.time() - -z = (y - x) / 60 # z is time for lopf in minutes - - - -#write results -network.model.write(args['outfile'], io_options={'symbolic_solver_labels':True}) - -#results_to_csv(network, args['results']) - -# plots - -# make a line loading plot -plot_line_loading(network) - -# plot stacked sum of nominal power for each generator type and timestep -plot_stacked_gen(network, resolution="MW") - -# plot to show extendable storages -storage_distribution(network) - -# close session -session.close() From 0c3ec9c550c3d418941ff4f12d3d159f2ee4442e Mon Sep 17 00:00:00 2001 From: Martin Date: Tue, 5 Sep 2017 15:26:23 +0200 Subject: [PATCH 43/65] Rewrite busmap database export --- etrago/cluster/networkclustering.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/etrago/cluster/networkclustering.py b/etrago/cluster/networkclustering.py index 084f0c34..36f7fb5b 100644 --- a/etrago/cluster/networkclustering.py +++ b/etrago/cluster/networkclustering.py @@ -214,8 +214,10 @@ def busmap_by_shortest_path(network, session, scn_name, fromlvl, tolvl, df.rename(columns={'source': 'bus0', 'target': 'bus1'}, inplace=True) df.set_index(['scn_name', 'bus0', 'bus1'], inplace=True) - df.to_sql('ego_grid_pf_hv_busmap', if_exists='append', - con=session.bind, schema='model_draft') + for i, d in df.reset_index().iterrows(): + session.add(EgoGridPfHvBusmap(**d.to_dict())) + + session.commit() return From 9c13b0fbc8d06348e4c8f8f622bd978c07194f28 Mon Sep 17 00:00:00 2001 From: Martin Date: Tue, 5 Sep 2017 15:27:47 +0200 Subject: [PATCH 44/65] Call DataFrame set_value in data_manipulation_sh --- etrago/extras/utilities.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/etrago/extras/utilities.py b/etrago/extras/utilities.py index afa6ffc0..d3b6d381 100644 --- a/etrago/extras/utilities.py +++ b/etrago/extras/utilities.py @@ -144,16 +144,17 @@ def data_manipulation_sh (network): #bus geom point_bus1 = Point(10.760835,53.909745) - network.buses.geom[new_bus]= from_shape(point_bus1, 4326) - + + network.buses.set_value(new_bus, 'geom', from_shape(point_bus1, 4326)) + #line geom/topo - network.lines.geom[new_line] = from_shape(MultiLineString([LineString([to_shape(network.buses.geom['26387']),point_bus1])]),4326) - network.lines.topo[new_line] = from_shape(LineString([to_shape(network.buses.geom['26387']),point_bus1]),4326) - + network.lines.set_value(new_line, 'geom', from_shape(MultiLineString([LineString([to_shape(network.buses.geom['26387']),point_bus1])]),4326)) + network.lines.set_value(new_line, 'topo', from_shape(LineString([to_shape(network.buses.geom['26387']),point_bus1]),4326)) + #trafo geom/topo - network.transformers.geom[new_trafo] = from_shape(MultiLineString([LineString([to_shape(network.buses.geom['25536']),point_bus1])]),4326) - network.transformers.topo[new_trafo] = from_shape(LineString([to_shape(network.buses.geom['25536']),point_bus1]),4326) - + network.transformers.set_value(new_trafo, 'geom', from_shape(MultiLineString([LineString([to_shape(network.buses.geom['25536']),point_bus1])]),4326)) + network.transformers.set_value(new_trafo, 'geom', from_shape(LineString([to_shape(network.buses.geom['25536']),point_bus1]),4326)) + # future way to add the geoms of the new components, currently bugged in pandas/shapely # #bus geom From ad49fba752b60f48e81ddf77d4871516eb94af19 Mon Sep 17 00:00:00 2001 From: Martin Date: Tue, 5 Sep 2017 15:35:19 +0200 Subject: [PATCH 45/65] Revert "Call DataFrame set_value in data_manipulation_sh" This reverts commit 9c13b0fbc8d06348e4c8f8f622bd978c07194f28. --- etrago/extras/utilities.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/etrago/extras/utilities.py b/etrago/extras/utilities.py index d3b6d381..afa6ffc0 100644 --- a/etrago/extras/utilities.py +++ b/etrago/extras/utilities.py @@ -144,17 +144,16 @@ def data_manipulation_sh (network): #bus geom point_bus1 = Point(10.760835,53.909745) - - network.buses.set_value(new_bus, 'geom', from_shape(point_bus1, 4326)) - + network.buses.geom[new_bus]= from_shape(point_bus1, 4326) + #line geom/topo - network.lines.set_value(new_line, 'geom', from_shape(MultiLineString([LineString([to_shape(network.buses.geom['26387']),point_bus1])]),4326)) - network.lines.set_value(new_line, 'topo', from_shape(LineString([to_shape(network.buses.geom['26387']),point_bus1]),4326)) - + network.lines.geom[new_line] = from_shape(MultiLineString([LineString([to_shape(network.buses.geom['26387']),point_bus1])]),4326) + network.lines.topo[new_line] = from_shape(LineString([to_shape(network.buses.geom['26387']),point_bus1]),4326) + #trafo geom/topo - network.transformers.set_value(new_trafo, 'geom', from_shape(MultiLineString([LineString([to_shape(network.buses.geom['25536']),point_bus1])]),4326)) - network.transformers.set_value(new_trafo, 'geom', from_shape(LineString([to_shape(network.buses.geom['25536']),point_bus1]),4326)) - + network.transformers.geom[new_trafo] = from_shape(MultiLineString([LineString([to_shape(network.buses.geom['25536']),point_bus1])]),4326) + network.transformers.topo[new_trafo] = from_shape(LineString([to_shape(network.buses.geom['25536']),point_bus1]),4326) + # future way to add the geoms of the new components, currently bugged in pandas/shapely # #bus geom From ef785919efe90237d025f69d12a9d0da1918bf53 Mon Sep 17 00:00:00 2001 From: Martin Date: Tue, 5 Sep 2017 15:27:47 +0200 Subject: [PATCH 46/65] Call DataFrame set_value in data_manipulation_sh --- etrago/extras/utilities.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/etrago/extras/utilities.py b/etrago/extras/utilities.py index afa6ffc0..d3b6d381 100644 --- a/etrago/extras/utilities.py +++ b/etrago/extras/utilities.py @@ -144,16 +144,17 @@ def data_manipulation_sh (network): #bus geom point_bus1 = Point(10.760835,53.909745) - network.buses.geom[new_bus]= from_shape(point_bus1, 4326) - + + network.buses.set_value(new_bus, 'geom', from_shape(point_bus1, 4326)) + #line geom/topo - network.lines.geom[new_line] = from_shape(MultiLineString([LineString([to_shape(network.buses.geom['26387']),point_bus1])]),4326) - network.lines.topo[new_line] = from_shape(LineString([to_shape(network.buses.geom['26387']),point_bus1]),4326) - + network.lines.set_value(new_line, 'geom', from_shape(MultiLineString([LineString([to_shape(network.buses.geom['26387']),point_bus1])]),4326)) + network.lines.set_value(new_line, 'topo', from_shape(LineString([to_shape(network.buses.geom['26387']),point_bus1]),4326)) + #trafo geom/topo - network.transformers.geom[new_trafo] = from_shape(MultiLineString([LineString([to_shape(network.buses.geom['25536']),point_bus1])]),4326) - network.transformers.topo[new_trafo] = from_shape(LineString([to_shape(network.buses.geom['25536']),point_bus1]),4326) - + network.transformers.set_value(new_trafo, 'geom', from_shape(MultiLineString([LineString([to_shape(network.buses.geom['25536']),point_bus1])]),4326)) + network.transformers.set_value(new_trafo, 'geom', from_shape(LineString([to_shape(network.buses.geom['25536']),point_bus1]),4326)) + # future way to add the geoms of the new components, currently bugged in pandas/shapely # #bus geom From 65dd81faaaab7a7639bb11f4a5926a8aeb36eb75 Mon Sep 17 00:00:00 2001 From: wolfbunke Date: Tue, 5 Sep 2017 18:51:05 +0200 Subject: [PATCH 47/65] adapted weighting by gen and load to Kmean --- etrago/k_means_testing.py | 278 +++++++++++++++++++------------------- 1 file changed, 139 insertions(+), 139 deletions(-) diff --git a/etrago/k_means_testing.py b/etrago/k_means_testing.py index 4557f91c..8accdb0b 100644 --- a/etrago/k_means_testing.py +++ b/etrago/k_means_testing.py @@ -50,150 +50,150 @@ args = scenario_setting - -# function for k-mean ToDo: move to cluster.networkclustering or extras.utilities -def weighting_for_scenario(x): - b_i = x.index - g = normed(gen.reindex(b_i, fill_value=0)) - l = normed(load.reindex(b_i, fill_value=0)) - - w= g + l - return (w * (100. / w.max())).astype(int) - -def normed(x): - return (x/x.sum()).fillna(0.) - - - def etrago(args): - session = oedb_session(args['db']) - - # additional arguments cfgpath, version, prefix - scenario = NetworkScenario(session, - version=args['gridversion'], - prefix=args['ormcls_prefix'], - method=args['method'], - start_h=args['start_h'], - end_h=args['end_h'], - scn_name=args['scn_name']) - - network = scenario.build_network() - - # add coordinates - network = add_coordinates(network) - - # TEMPORARY vague adjustment due to transformer bug in data processing - #network.transformers.x=network.transformers.x*0.01 - - - if args['branch_capacity_factor']: - network.lines.s_nom = network.lines.s_nom*args['branch_capacity_factor'] - network.transformers.s_nom = network.transformers.s_nom*args['branch_capacity_factor'] - - if args['generator_noise']: - # create generator noise - noise_values = network.generators.marginal_cost + abs(np.random.normal(0,0.001,len(network.generators.marginal_cost))) - np.savetxt("noise_values.csv", noise_values, delimiter=",") - noise_values = genfromtxt('noise_values.csv', delimiter=',') - # add random noise to all generator - network.generators.marginal_cost = noise_values - - if args['storage_extendable']: - # set virtual storages to be extendable - if network.storage_units.source.any()=='extendable_storage': - network.storage_units.p_nom_extendable = True - # set virtual storage costs with regards to snapshot length - network.storage_units.capital_cost = (network.storage_units.capital_cost / - (8760//(args['end_h']-args['start_h']+1))) - - # for SH scenario run do data preperation: - if args['scn_name'] == 'SH Status Quo': - data_manipulation_sh(network) - - #load shedding in order to hunt infeasibilities - if args['load_shedding']: - load_shedding(network) - - # network clustering - if args['network_clustering']: - network.generators.control="PV" - busmap = busmap_from_psql(network, session, scn_name=args['scn_name']) - network = cluster_on_extra_high_voltage(network, busmap, with_time=True) - - - - if args['k_mean_clustering']: - print('start k-mean clustering') - # prepare k-mean - # k-means clustering (first try) - network.generators.control="PV" - network.buses['v_nom'] = 380. - # problem our lines have no v_nom. this is implicitly defined by the connected buses: - network.lines["v_nom"] = network.lines.bus0.map(network.buses.v_nom) - - # adjust the x of the lines which are not 380. - lines_v_nom_b = network.lines.v_nom != 380 - network.lines.loc[lines_v_nom_b, 'x'] *= (380./network.lines.loc[lines_v_nom_b, 'v_nom'])**2 - network.lines.loc[lines_v_nom_b, 'v_nom'] = 380. - - trafo_index = network.transformers.index - - network.import_components_from_dataframe( - network.transformers.loc[:,['bus0','bus1','x','s_nom']] - .assign(x=0.1*380**2/2000) - .set_index('T' + trafo_index), - 'Line') - network.transformers.drop(trafo_index, inplace=True) - - for attr in network.transformers_t: - network.transformers_t[attr] = network.transformers_t[attr].reindex(columns=[]) - - # ToDo: - #change np.repeat(1, len(network.buses) to load and conv P_max - conv_types = {'biomass', 'run_of_river', 'gas', 'oil','coal', 'waste','uranium'} - # Attention: network.generators.carrier.unique() - # conv_types only for SH scenario defined! - gen = (network.generators.loc[network.generators.carrier.isin(conv_types) + + def weighting_for_scenario(x): + b_i = x.index + g = normed(gen.reindex(b_i, fill_value=0)) + l = normed(load.reindex(b_i, fill_value=0)) + + w= g + l + return (w * (100. / w.max())).astype(int) + + def normed(x): + return (x/x.sum()).fillna(0.) + + + session = oedb_session(args['db']) + + # additional arguments cfgpath, version, prefix + scenario = NetworkScenario(session, + version=args['gridversion'], + prefix=args['ormcls_prefix'], + method=args['method'], + start_h=args['start_h'], + end_h=args['end_h'], + scn_name=args['scn_name']) + + network = scenario.build_network() + + # add coordinates + network = add_coordinates(network) + + # TEMPORARY vague adjustment due to transformer bug in data processing + #network.transformers.x=network.transformers.x*0.01 + + + if args['branch_capacity_factor']: + network.lines.s_nom = network.lines.s_nom*args['branch_capacity_factor'] + network.transformers.s_nom = network.transformers.s_nom*args['branch_capacity_factor'] + + if args['generator_noise']: + # create generator noise + noise_values = network.generators.marginal_cost + abs(np.random.normal(0,0.001,len(network.generators.marginal_cost))) + np.savetxt("noise_values.csv", noise_values, delimiter=",") + noise_values = genfromtxt('noise_values.csv', delimiter=',') + # add random noise to all generator + network.generators.marginal_cost = noise_values + + if args['storage_extendable']: + # set virtual storages to be extendable + if network.storage_units.source.any()=='extendable_storage': + network.storage_units.p_nom_extendable = True + # set virtual storage costs with regards to snapshot length + network.storage_units.capital_cost = (network.storage_units.capital_cost / + (8760//(args['end_h']-args['start_h']+1))) + + # for SH scenario run do data preperation: + if args['scn_name'] == 'SH Status Quo': + data_manipulation_sh(network) + + #load shedding in order to hunt infeasibilities + if args['load_shedding']: + load_shedding(network) + + # network clustering + if args['network_clustering']: + network.generators.control="PV" + busmap = busmap_from_psql(network, session, scn_name=args['scn_name']) + network = cluster_on_extra_high_voltage(network, busmap, with_time=True) + + + + if args['k_mean_clustering']: + print('start k-mean clustering') + # prepare k-mean + # k-means clustering (first try) + network.generators.control="PV" + network.buses['v_nom'] = 380. + # problem our lines have no v_nom. this is implicitly defined by the connected buses: + network.lines["v_nom"] = network.lines.bus0.map(network.buses.v_nom) + + # adjust the x of the lines which are not 380. + lines_v_nom_b = network.lines.v_nom != 380 + network.lines.loc[lines_v_nom_b, 'x'] *= (380./network.lines.loc[lines_v_nom_b, 'v_nom'])**2 + network.lines.loc[lines_v_nom_b, 'v_nom'] = 380. + + trafo_index = network.transformers.index + + network.import_components_from_dataframe( + network.transformers.loc[:,['bus0','bus1','x','s_nom']] + .assign(x=0.1*380**2/2000) + .set_index('T' + trafo_index), + 'Line') + network.transformers.drop(trafo_index, inplace=True) + + for attr in network.transformers_t: + network.transformers_t[attr] = network.transformers_t[attr].reindex(columns=[]) + + # ToDo: + #change np.repeat(1, len(network.buses) to load and conv P_max + conv_types = {'biomass', 'run_of_river', 'gas', 'oil','coal', 'waste','uranium'} + # Attention: network.generators.carrier.unique() + # conv_types only for SH scenario defined! + gen = (network.generators.loc[network.generators.carrier.isin(conv_types) ].groupby('bus').p_nom.sum().reindex(network.buses.index, fill_value=0.) + network.storage_units.loc[network.storage_units.carrier.isin(conv_types) ].groupby('bus').p_nom.sum().reindex(network.buses.index, fill_value=0.)) - load = network.loads_t.p_set.mean().groupby(network.loads.bus).sum() - - # k-mean clustering - busmap = busmap_by_kmeans(network, bus_weightings=pd.Series(np.repeat(1, - len(network.buses)), index=network.buses.index) , n_clusters= 10) - - # ToDo change function in order to use bus_strategies or similar - clustering = get_clustering_from_busmap(network, busmap) - network = clustering.network - #network = cluster_on_extra_high_voltage(network, busmap, with_time=True) - - - # parallisation - if args['parallelisation']: - parallelisation(network, start_h=args['start_h'], end_h=args['end_h'],group_size=1, solver_name=args['solver']) - # start linear optimal powerflow calculations - elif args['method'] == 'lopf': - x = time.time() - network.lopf(scenario.timeindex, solver_name=args['solver']) - y = time.time() - z = (y - x) / 60 # z is time for lopf in minutes - # start non-linear powerflow simulation - elif args['method'] == 'pf': - network.pf(scenario.timeindex) - if args['pf_post_lopf']: - pf_post_lopf(network, scenario) - - # write lpfile to path - if not args['lpfile'] == False: - network.model.write(args['lpfile'], io_options={'symbolic_solver_labels': - True}) - # write PyPSA results to csv to path - if not args['results'] == False: - results_to_csv(network, args['results']) - - return network + load = network.loads_t.p_set.mean().groupby(network.loads.bus).sum() + + # k-mean clustering + # busmap = busmap_by_kmeans(network, bus_weightings=pd.Series(np.repeat(1, + # len(network.buses)), index=network.buses.index) , n_clusters= 10) + weight = weighting_for_scenario(network.buses).reindex(network.buses.index, fill_value=1) + busmap = busmap_by_kmeans(network, bus_weightings=pd.Series(weight), buses_i=network.buses.index , n_clusters= 10) + + + # ToDo change function in order to use bus_strategies or similar + clustering = get_clustering_from_busmap(network, busmap) + network = clustering.network + #network = cluster_on_extra_high_voltage(network, busmap, with_time=True) + + + # parallisation + if args['parallelisation']: + parallelisation(network, start_h=args['start_h'], end_h=args['end_h'],group_size=1, solver_name=args['solver']) + # start linear optimal powerflow calculations + elif args['method'] == 'lopf': + x = time.time() + network.lopf(scenario.timeindex, solver_name=args['solver']) + y = time.time() + z = (y - x) / 60 # z is time for lopf in minutes + # start non-linear powerflow simulation + elif args['method'] == 'pf': + network.pf(scenario.timeindex) + if args['pf_post_lopf']: + pf_post_lopf(network, scenario) + + # write lpfile to path + if not args['lpfile'] == False: + network.model.write(args['lpfile'], io_options={'symbolic_solver_labels': True}) + # write PyPSA results to csv to path + if not args['results'] == False: + results_to_csv(network, args['results']) + + return network # execute etrago function network = etrago(args) From 5f96d71e4dc378ad01cd0d1ce723f9d19b498303 Mon Sep 17 00:00:00 2001 From: MariusVes Date: Wed, 6 Sep 2017 09:30:09 +0200 Subject: [PATCH 48/65] Prior commit works, thus removing comments --- etrago/extras/utilities.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/etrago/extras/utilities.py b/etrago/extras/utilities.py index d3b6d381..04708c0b 100644 --- a/etrago/extras/utilities.py +++ b/etrago/extras/utilities.py @@ -144,7 +144,6 @@ def data_manipulation_sh (network): #bus geom point_bus1 = Point(10.760835,53.909745) - network.buses.set_value(new_bus, 'geom', from_shape(point_bus1, 4326)) #line geom/topo @@ -155,20 +154,6 @@ def data_manipulation_sh (network): network.transformers.set_value(new_trafo, 'geom', from_shape(MultiLineString([LineString([to_shape(network.buses.geom['25536']),point_bus1])]),4326)) network.transformers.set_value(new_trafo, 'geom', from_shape(LineString([to_shape(network.buses.geom['25536']),point_bus1]),4326)) -# future way to add the geoms of the new components, currently bugged in pandas/shapely - -# #bus geom -# point_bus1 = Point(10.760835,53.909745) -# network.buses.loc[new_bus,'geom'] = from_shape(point_bus1, 4326) -# -# #line geom/topo -# network.lines.loc[new_line,'geom'] = from_shape(MultiLineString([LineString([to_shape(network.buses.geom['26387']),point_bus1])]),4326) -# network.lines.loc[new_line,'topo'] = from_shape(LineString([to_shape(network.buses.geom['26387']),point_bus1]),4326) -# -# #trafo geom/topo -# network.transformers.loc[new_trafo,'geom'] = from_shape(MultiLineString([LineString([to_shape(network.buses.geom['25536']),point_bus1])]),4326) -# network.transformers.loc[new_trafo,'topo'] = from_shape(LineString([to_shape(network.buses.geom['25536']),point_bus1]),4326) - return def results_to_csv(network, path): From 6ee4fc4367f0a516ae2cdccde2ef71bc85ec9e9f Mon Sep 17 00:00:00 2001 From: ulf Date: Wed, 6 Sep 2017 18:25:05 +0200 Subject: [PATCH 49/65] changed tranformer x calculation --- etrago/k_means_testing.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/etrago/k_means_testing.py b/etrago/k_means_testing.py index 8accdb0b..8c903503 100644 --- a/etrago/k_means_testing.py +++ b/etrago/k_means_testing.py @@ -135,10 +135,12 @@ def normed(x): network.lines.loc[lines_v_nom_b, 'v_nom'] = 380. trafo_index = network.transformers.index + transformer_voltages = pd.concat([network.transformers.bus0.map(network.buses.v_nom), network.transformers.bus1.map(network.buses.v_nom)], axis=1) + network.import_components_from_dataframe( network.transformers.loc[:,['bus0','bus1','x','s_nom']] - .assign(x=0.1*380**2/2000) + .assign(x=network.transformers.x*(380./transformer_voltages.max(axis=1)**2)) .set_index('T' + trafo_index), 'Line') network.transformers.drop(trafo_index, inplace=True) From 69a915909937bd1bf35f983f4318b97f84e1ff3f Mon Sep 17 00:00:00 2001 From: ulf Date: Wed, 6 Sep 2017 18:34:06 +0200 Subject: [PATCH 50/65] minor bug fix --- etrago/k_means_testing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etrago/k_means_testing.py b/etrago/k_means_testing.py index 8c903503..2889b9a9 100644 --- a/etrago/k_means_testing.py +++ b/etrago/k_means_testing.py @@ -140,7 +140,7 @@ def normed(x): network.import_components_from_dataframe( network.transformers.loc[:,['bus0','bus1','x','s_nom']] - .assign(x=network.transformers.x*(380./transformer_voltages.max(axis=1)**2)) + .assign(x=network.transformers.x*(380./transformer_voltages.max(axis=1))**2) .set_index('T' + trafo_index), 'Line') network.transformers.drop(trafo_index, inplace=True) From bda27bc99ab59d22e67bfe76d94043797e1c49d4 Mon Sep 17 00:00:00 2001 From: mariusves Date: Thu, 7 Sep 2017 13:00:15 +0200 Subject: [PATCH 51/65] changed name of start_h and end_h variables also see changes made in the ego.powerflow dev branch! --- etrago/appl.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/etrago/appl.py b/etrago/appl.py index 4b69168b..5528456a 100644 --- a/etrago/appl.py +++ b/etrago/appl.py @@ -27,8 +27,8 @@ 'gridversion': 'v0.2.11', #None for model_draft or Version number (e.g. v0.2.10) for grid schema 'method': 'lopf', # lopf or pf 'pf_post_lopf':False , #state whether you want to perform a pf after a lopf simulation - 'start_h': 2320, - 'end_h' : 2321, + 'start_snapshot': 2320, + 'end_snapshot' : 2321, 'scn_name': 'SH Status Quo', 'ormcls_prefix': 'EgoPfHv', #if gridversion:'version-number' then 'EgoPfHv', if gridversion:None then 'EgoGridPfHv' 'lpfile': False, # state if and where you want to save pyomo's lp file: False or '/path/tofolder/file.lp' @@ -51,8 +51,8 @@ def etrago(args): version=args['gridversion'], prefix=args['ormcls_prefix'], method=args['method'], - start_h=args['start_h'], - end_h=args['end_h'], + start_snapshot=args['start_snapshot'], + end_snapshot=args['end_snapshot'], scn_name=args['scn_name']) network = scenario.build_network() @@ -82,7 +82,7 @@ def etrago(args): network.storage_units.p_nom_extendable = True # set virtual storage costs with regards to snapshot length network.storage_units.capital_cost = (network.storage_units.capital_cost / - (8760//(args['end_h']-args['start_h']+1))) + (8760//(args['end_snapshot']-args['start_snapshot']+1))) # for SH scenario run do data preperation: if args['scn_name'] == 'SH Status Quo': @@ -106,7 +106,7 @@ def etrago(args): # parallisation if args['parallelisation']: - parallelisation(network, start_h=args['start_h'], end_h=args['end_h'],group_size=1, solver_name=args['solver'], extra_functionality=extra_functionality) + parallelisation(network, start_snapshot=args['start_snapshot'], end_snapshot=args['end_snapshot'],group_size=1, solver_name=args['solver'], extra_functionality=extra_functionality) # start linear optimal powerflow calculations elif args['method'] == 'lopf': x = time.time() From e4eb9b15c0e28f5daef6f97db2a4551f77e5d39a Mon Sep 17 00:00:00 2001 From: mariusves Date: Thu, 7 Sep 2017 13:09:39 +0200 Subject: [PATCH 52/65] added ormclass check for gridversion (see #39) --- etrago/appl.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/etrago/appl.py b/etrago/appl.py index 5528456a..3474aac0 100644 --- a/etrago/appl.py +++ b/etrago/appl.py @@ -30,7 +30,6 @@ 'start_snapshot': 2320, 'end_snapshot' : 2321, 'scn_name': 'SH Status Quo', - 'ormcls_prefix': 'EgoPfHv', #if gridversion:'version-number' then 'EgoPfHv', if gridversion:None then 'EgoGridPfHv' 'lpfile': False, # state if and where you want to save pyomo's lp file: False or '/path/tofolder/file.lp' 'results': False, # state if and where you want to save results as csv: False or '/path/tofolder' 'export': False, # state if you want to export the results back to the database @@ -47,6 +46,11 @@ def etrago(args): session = oedb_session(args['db']) # additional arguments cfgpath, version, prefix + if args['gridversion'] == None: + args['ormcls_prefix'] = 'EgoGridPfHv' + else: + args['ormcls_prefix'] = 'EgoPfHv' + scenario = NetworkScenario(session, version=args['gridversion'], prefix=args['ormcls_prefix'], From 5602b66df2b4bc47773602ce20195c8db1dfefbd Mon Sep 17 00:00:00 2001 From: Lukas Date: Thu, 7 Sep 2017 14:50:06 +0200 Subject: [PATCH 53/65] include first approaches to loss calculation #23 --- etrago/appl.py | 25 +++++++++++++++++-------- etrago/extras/utilities.py | 31 ++++++++++++++++++++++--------- 2 files changed, 39 insertions(+), 17 deletions(-) diff --git a/etrago/appl.py b/etrago/appl.py index c47445b3..391ab418 100644 --- a/etrago/appl.py +++ b/etrago/appl.py @@ -27,16 +27,16 @@ 'gridversion':'v0.2.11', #None for model_draft or Version number (e.g. v0.2.10) for grid schema 'method': 'lopf', # lopf or pf 'pf_post_lopf': True, #state whether you want to perform a pf after a lopf simulation - 'start_h': 2320, - 'end_h' : 2325, - 'scn_name': 'SH Status Quo', + 'start_h': 1, + 'end_h' : 12, + 'scn_name': 'SH NEP 2035', 'ormcls_prefix': 'EgoPfHv', #if gridversion:'version-number' then 'EgoPfHv', if gridversion:None then 'EgoGridPfHv' 'lpfile': False, # state if and where you want to save pyomo's lp file: False or '/path/tofolder' - 'results': '/srv/ES2050/open_eGo/AP3/lopf_results/gurobi_sh_pf_1' , # state if and where you want to save results as csv: False or '/path/tofolder' + 'results': False , # state if and where you want to save results as csv: False or '/path/tofolder' 'solver': 'gurobi', #glpk, cplex or gurobi 'branch_capacity_factor': 1, #to globally extend or lower branch capacities - 'storage_extendable':False, - 'load_shedding':True, + 'storage_extendable':True, + 'load_shedding':False, 'generator_noise':False, 'parallelisation':False} @@ -95,7 +95,9 @@ def etrago(args): network.generators.control="PV" busmap = busmap_from_psql(network, session, scn_name=args['scn_name']) network = cluster_on_extra_high_voltage(network, busmap, with_time=True) - + + # network.snapshot_weightings[3]=3 + # parallisation if args['parallelisation']: parallelisation(network, start_h=args['start_h'], end_h=args['end_h'],group_size=1, solver_name=args['solver']) @@ -108,11 +110,18 @@ def etrago(args): # start non-linear powerflow simulation elif args['method'] == 'pf': network.pf(scenario.timeindex) + # calc_line_losses(network) if args['pf_post_lopf']: pf_post_lopf(network, scenario) calc_line_losses(network) - + + # provide storage installation costs + if sum(network.storage_units.p_nom_opt) != 0: + installed_storages = network.storage_units[ network.storage_units.p_nom_opt!=0] + storage_costs = sum(installed_storages.capital_cost * installed_storages.p_nom_opt) + print("Investment costs for all storages in selected snapshots [EUR]:",round(storage_costs,2)) + # write lpfile to path if not args['lpfile'] == False: network.model.write(args['lpfile'], io_options={'symbolic_solver_labels': diff --git a/etrago/extras/utilities.py b/etrago/extras/utilities.py index 8e21f142..5f315664 100644 --- a/etrago/extras/utilities.py +++ b/etrago/extras/utilities.py @@ -207,20 +207,33 @@ def calc_line_losses(network): ------- """ - + #### Line losses # calculate apparent power S = sqrt(p² + q²) - s0 = ((network.lines_t.p0**2 + network.lines_t.q0**2).\ + s0_lines = ((network.lines_t.p0**2 + network.lines_t.q0**2).\ apply(np.sqrt)) # calculate current I = S / U - i0 = s0 / network.lines.v_nom + i0_lines = s0_lines / network.lines.v_nom # calculate losses per line and timestep network.lines_t.line_losses = I² * R - p_loss_t = i0**2 * network.lines.r - network.lines_t.line_losses = p_loss_t + network.lines_t.losses = i0_lines**2 * network.lines.r + # calculate total losses per line + network.lines.losses = np.sum(network.lines_t.losses) + + #### Transformer losses + # calculate apparent power S = sqrt(p² + q²) + s0_trafo = ((network.transformers_t.p0**2 + network.transformers_t.q0**2).\ + apply(np.sqrt)) + # calculate losses per transformer and timestep + # network.transformers_t.losses = s0_trafo / network.transformers.s_nom ## !!! this needs to be finalised + # calculate fix no-load losses per transformer + network.transformers.losses_fix = 0.00275 * network.transformers.s_nom # average value according to http://ibn.ch/HomePageSchule/Schule/GIBZ/19_Transformatoren/19_Transformatoren_Loesung.pdf # calculate total losses per line - p_loss = np.sum(p_loss_t) - # calculate total losses on all lines - p_loss_overall = sum(p_loss) - print("Total lines losses for all snapshots and lines [MW]:",p_loss_overall) + network.transformers.losses = network.transformers.losses_fix # + np.sum(network.transformers_t.losses) + + # calculate total losses (possibly enhance with adding these values to network container) + losses_total = sum(network.lines.losses) + sum(network.transformers.losses) + print("Total lines losses for all snapshots [MW]:",round(losses_total,2)) + losses_costs = losses_total * np.average(network.buses_t.marginal_price) + print("Total costs for these losses [EUR]:",round(losses_costs,2)) return From 08ad797dfac80e97ac636ed13cdbc8853922a466 Mon Sep 17 00:00:00 2001 From: wolfbunke Date: Fri, 8 Sep 2017 07:53:51 +0200 Subject: [PATCH 54/65] delete testing files --- etrago/k_means_testing.py | 221 ----------------------------------- etrago/scenario_setting.json | 21 ---- 2 files changed, 242 deletions(-) delete mode 100644 etrago/k_means_testing.py delete mode 100644 etrago/scenario_setting.json diff --git a/etrago/k_means_testing.py b/etrago/k_means_testing.py deleted file mode 100644 index 2889b9a9..00000000 --- a/etrago/k_means_testing.py +++ /dev/null @@ -1,221 +0,0 @@ -""" -K-means testing File - -see: https://github.com/openego/eTraGo/issues/6 - -ToDo's: -------- -the remaining work would be: - -- [x] implement the [todo](https://github.com/openego/eTraGo/blob/features/k-means-clustering/etrago/k_means_testing.py#L112-L115) so that the x of the lines which are newly defined as 380kV lines are adjusted - -- [ ] in the [Hoersch and Brown contribution](https://arxiv.org/pdf/1705.07617.pdf) in Chapter II 2) and follwoing the weighting is defined. the weighting right now is equal over all buses. This should be changed to the assumptions with respect to the contribution or define another senseful weighting - -- [ ] add functionality to save the resulting cluster for reproducibility - -- [ ] convert it to a function and move it [here](https://github.com/openego/eTraGo/blob/features/k-means-clustering/etrago/cluster/networkclustering.py) - -Error handling: - -* use pip3 install scikit-learn in order to use PyPSA busmap_by_kmeans - - - - -""" -__copyright__ = "tba" -__license__ = "tba" -__author__ = "tba" - - -import numpy as np -from numpy import genfromtxt -np.random.seed() -from egopowerflow.tools.tools import oedb_session -from egopowerflow.tools.io import NetworkScenario -import time -from egopowerflow.tools.plot import (plot_line_loading, plot_stacked_gen, - add_coordinates, curtailment, gen_dist, - storage_distribution) -from etrago.extras.utilities import load_shedding, data_manipulation_sh, results_to_csv, parallelisation, pf_post_lopf -from etrago.cluster.networkclustering import busmap_from_psql, cluster_on_extra_high_voltage -from pypsa.networkclustering import busmap_by_kmeans, get_clustering_from_busmap -import pandas as pd -import json - - -# import scenario settings **args -with open('scenario_setting.json') as f: - scenario_setting = json.load(f) - -args = scenario_setting - -def etrago(args): - - def weighting_for_scenario(x): - b_i = x.index - g = normed(gen.reindex(b_i, fill_value=0)) - l = normed(load.reindex(b_i, fill_value=0)) - - w= g + l - return (w * (100. / w.max())).astype(int) - - def normed(x): - return (x/x.sum()).fillna(0.) - - - session = oedb_session(args['db']) - - # additional arguments cfgpath, version, prefix - scenario = NetworkScenario(session, - version=args['gridversion'], - prefix=args['ormcls_prefix'], - method=args['method'], - start_h=args['start_h'], - end_h=args['end_h'], - scn_name=args['scn_name']) - - network = scenario.build_network() - - # add coordinates - network = add_coordinates(network) - - # TEMPORARY vague adjustment due to transformer bug in data processing - #network.transformers.x=network.transformers.x*0.01 - - - if args['branch_capacity_factor']: - network.lines.s_nom = network.lines.s_nom*args['branch_capacity_factor'] - network.transformers.s_nom = network.transformers.s_nom*args['branch_capacity_factor'] - - if args['generator_noise']: - # create generator noise - noise_values = network.generators.marginal_cost + abs(np.random.normal(0,0.001,len(network.generators.marginal_cost))) - np.savetxt("noise_values.csv", noise_values, delimiter=",") - noise_values = genfromtxt('noise_values.csv', delimiter=',') - # add random noise to all generator - network.generators.marginal_cost = noise_values - - if args['storage_extendable']: - # set virtual storages to be extendable - if network.storage_units.source.any()=='extendable_storage': - network.storage_units.p_nom_extendable = True - # set virtual storage costs with regards to snapshot length - network.storage_units.capital_cost = (network.storage_units.capital_cost / - (8760//(args['end_h']-args['start_h']+1))) - - # for SH scenario run do data preperation: - if args['scn_name'] == 'SH Status Quo': - data_manipulation_sh(network) - - #load shedding in order to hunt infeasibilities - if args['load_shedding']: - load_shedding(network) - - # network clustering - if args['network_clustering']: - network.generators.control="PV" - busmap = busmap_from_psql(network, session, scn_name=args['scn_name']) - network = cluster_on_extra_high_voltage(network, busmap, with_time=True) - - - - if args['k_mean_clustering']: - print('start k-mean clustering') - # prepare k-mean - # k-means clustering (first try) - network.generators.control="PV" - network.buses['v_nom'] = 380. - # problem our lines have no v_nom. this is implicitly defined by the connected buses: - network.lines["v_nom"] = network.lines.bus0.map(network.buses.v_nom) - - # adjust the x of the lines which are not 380. - lines_v_nom_b = network.lines.v_nom != 380 - network.lines.loc[lines_v_nom_b, 'x'] *= (380./network.lines.loc[lines_v_nom_b, 'v_nom'])**2 - network.lines.loc[lines_v_nom_b, 'v_nom'] = 380. - - trafo_index = network.transformers.index - transformer_voltages = pd.concat([network.transformers.bus0.map(network.buses.v_nom), network.transformers.bus1.map(network.buses.v_nom)], axis=1) - - - network.import_components_from_dataframe( - network.transformers.loc[:,['bus0','bus1','x','s_nom']] - .assign(x=network.transformers.x*(380./transformer_voltages.max(axis=1))**2) - .set_index('T' + trafo_index), - 'Line') - network.transformers.drop(trafo_index, inplace=True) - - for attr in network.transformers_t: - network.transformers_t[attr] = network.transformers_t[attr].reindex(columns=[]) - - # ToDo: - #change np.repeat(1, len(network.buses) to load and conv P_max - conv_types = {'biomass', 'run_of_river', 'gas', 'oil','coal', 'waste','uranium'} - # Attention: network.generators.carrier.unique() - # conv_types only for SH scenario defined! - gen = (network.generators.loc[network.generators.carrier.isin(conv_types) - ].groupby('bus').p_nom.sum().reindex(network.buses.index, - fill_value=0.) + network.storage_units.loc[network.storage_units.carrier.isin(conv_types) - ].groupby('bus').p_nom.sum().reindex(network.buses.index, fill_value=0.)) - - load = network.loads_t.p_set.mean().groupby(network.loads.bus).sum() - - # k-mean clustering - # busmap = busmap_by_kmeans(network, bus_weightings=pd.Series(np.repeat(1, - # len(network.buses)), index=network.buses.index) , n_clusters= 10) - weight = weighting_for_scenario(network.buses).reindex(network.buses.index, fill_value=1) - busmap = busmap_by_kmeans(network, bus_weightings=pd.Series(weight), buses_i=network.buses.index , n_clusters= 10) - - - # ToDo change function in order to use bus_strategies or similar - clustering = get_clustering_from_busmap(network, busmap) - network = clustering.network - #network = cluster_on_extra_high_voltage(network, busmap, with_time=True) - - - # parallisation - if args['parallelisation']: - parallelisation(network, start_h=args['start_h'], end_h=args['end_h'],group_size=1, solver_name=args['solver']) - # start linear optimal powerflow calculations - elif args['method'] == 'lopf': - x = time.time() - network.lopf(scenario.timeindex, solver_name=args['solver']) - y = time.time() - z = (y - x) / 60 # z is time for lopf in minutes - # start non-linear powerflow simulation - elif args['method'] == 'pf': - network.pf(scenario.timeindex) - if args['pf_post_lopf']: - pf_post_lopf(network, scenario) - - # write lpfile to path - if not args['lpfile'] == False: - network.model.write(args['lpfile'], io_options={'symbolic_solver_labels': True}) - # write PyPSA results to csv to path - if not args['results'] == False: - results_to_csv(network, args['results']) - - return network - -# execute etrago function -network = etrago(args) - - -#network.plot(bus_sizes=(2*weighting_for_scenario(network.buses.query('scn_name=="SH Status Quo"'))).reindex(network.buses.index, fill_value=1)) - - -# plots -# make a line loading plot -plot_line_loading(network) - -#plot stacked sum of nominal power for each generator type and timestep -#plot_stacked_gen(network, resolution="MW") - -# close session -#session.close() - -# plot stacked sum of nominal power for each generator type and timestep -plot_stacked_gen(network, resolution="MW") - -# plot to show extendable storages -storage_distribution(network) diff --git a/etrago/scenario_setting.json b/etrago/scenario_setting.json deleted file mode 100644 index b4a9f242..00000000 --- a/etrago/scenario_setting.json +++ /dev/null @@ -1,21 +0,0 @@ -{ -"network_clustering": false, -"db": "oedb", -"gridversion": "v0.2.11", -"method": "lopf", -"pf_post_lopf": false, -"start_h": 2323, -"end_h" : 2324, -"scn_name": "SH Status Quo", -"ormcls_prefix": "EgoPfHv", -"lpfile": "/home/dozeumbuw/eTraGo/src/etrago/etrago/results/file.lp", -"results": "/home/dozeumbuw/eTraGo/src/etrago/etrago/results/", -"solver": "gurobi", -"branch_capacity_factor": 1, -"storage_extendable": true, -"load_shedding": true, -"generator_noise": true, -"parallelisation": false, -"k_mean_clustering": true -} - From 65f0c31bc3a795e9644738ef832dd05b56aed8b9 Mon Sep 17 00:00:00 2001 From: WolfBunke Date: Fri, 8 Sep 2017 09:19:03 +0200 Subject: [PATCH 55/65] add network assignment of k-mean --- etrago/appl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etrago/appl.py b/etrago/appl.py index bce106e8..12bd03f3 100644 --- a/etrago/appl.py +++ b/etrago/appl.py @@ -115,7 +115,7 @@ def etrago(args): # k-mean clustering if args['k_mean_clustering']: - kmean_clustering(network) + network = kmean_clustering(network) # Branch loading minimization if args['minimize_loading']: From ce04cfcf83cd8c19e3fea918e593d2e9b95be34f Mon Sep 17 00:00:00 2001 From: mariusves Date: Fri, 8 Sep 2017 09:49:56 +0200 Subject: [PATCH 56/65] added grouping of parallel lines --- etrago/appl.py | 11 +++++++--- etrago/extras/utilities.py | 44 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 3 deletions(-) diff --git a/etrago/appl.py b/etrago/appl.py index 12bd03f3..2b1f078c 100644 --- a/etrago/appl.py +++ b/etrago/appl.py @@ -20,7 +20,7 @@ add_coordinates, curtailment, gen_dist, storage_distribution) -from etrago.extras.utilities import load_shedding, data_manipulation_sh, results_to_csv, parallelisation, pf_post_lopf, loading_minimization, calc_line_losses,kmean_clustering +from etrago.extras.utilities import load_shedding, data_manipulation_sh, results_to_csv, parallelisation, pf_post_lopf, loading_minimization, calc_line_losses, kmean_clustering, group_parallel_lines from etrago.cluster.networkclustering import busmap_from_psql, cluster_on_extra_high_voltage from pypsa.networkclustering import busmap_by_kmeans, get_clustering_from_busmap @@ -45,7 +45,8 @@ 'minimize_loading':False, 'k_mean_clustering': True, 'parallelisation':False, - 'comments': None} + 'line_grouping': False, + 'comments': None} def etrago(args): @@ -122,7 +123,11 @@ def etrago(args): extra_functionality = loading_minimization else: extra_functionality=None - + + # grouping of parallel lines + if args['line_grouping']: + group_parallel_lines(network) + # parallisation if args['parallelisation']: parallelisation(network, start_snapshot=args['start_snapshot'], end_snapshot=args['end_snapshot'],group_size=1, solver_name=args['solver'], extra_functionality=extra_functionality) diff --git a/etrago/extras/utilities.py b/etrago/extras/utilities.py index f111d938..e69e21fc 100644 --- a/etrago/extras/utilities.py +++ b/etrago/extras/utilities.py @@ -346,7 +346,51 @@ def normed(x): return network +def group_parallel_lines(network): + old_lines = network.lines + for line in old_lines: + old_lines['old_index'] = network.lines.index + + grouped = old_lines.groupby(['bus0','bus1']) + + grouped_agg = grouped.agg({ 'b': np.sum, + 'b_pu': np.sum, + 'cables': np.sum, + 'capital_cost': np.min, + 'frequency': np.mean, + 'g': np.sum, + 'g_pu': np.sum, + 'geom': lambda x: x[0], + 'length': lambda x: x.min(), + 'num_parallel': np.sum, + 'r': lambda x: np.reciprocal(np.sum(np.reciprocal(x))), + 'r_pu': lambda x: np.reciprocal(np.sum(np.reciprocal(x))), + 's_nom': np.sum, + 's_nom_extendable': lambda x: x.min(), + 's_nom_max': np.sum, + 's_nom_min': np.sum, + 's_nom_opt': np.sum, + 'scn_name': lambda x: x.min(), + 'sub_network': lambda x: x.min(), + 'terrain_factor': lambda x: x.min(), + 'topo': lambda x: x[0], + 'type': lambda x: x.min(), + 'v_ang_max': lambda x: x.min(), + 'v_ang_min': lambda x: x.min(), + 'x': lambda x: np.reciprocal(np.sum(np.reciprocal(x))), + 'x_pu': lambda x: np.reciprocal(np.sum(np.reciprocal(x))), + 'old_index': np.min}) + + for i in range(0,len(grouped_agg.index)): + grouped_agg.set_value(grouped_agg.index[i],'bus0',grouped_agg.index[i][0]) + grouped_agg.set_value(grouped_agg.index[i],'bus1',grouped_agg.index[i][1]) + + new_lines=grouped_agg.set_index(grouped_agg.old_index) + new_lines=new_lines.drop('old_index',1) + network.lines = new_lines + + return From ce1226974010c3d76b2b8473bcc3e202a677fec2 Mon Sep 17 00:00:00 2001 From: MariusVes Date: Fri, 8 Sep 2017 09:56:48 +0200 Subject: [PATCH 57/65] changed position of grouping function in script otherwise there might be conflicts with network clustering. --- etrago/appl.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/etrago/appl.py b/etrago/appl.py index 2b1f078c..7ee08c7e 100644 --- a/etrago/appl.py +++ b/etrago/appl.py @@ -103,6 +103,10 @@ def etrago(args): # for SH scenario run do data preperation: if args['scn_name'] == 'SH Status Quo': data_manipulation_sh(network) + + # grouping of parallel lines + if args['line_grouping']: + group_parallel_lines(network) #load shedding in order to hunt infeasibilities if args['load_shedding']: @@ -124,10 +128,6 @@ def etrago(args): else: extra_functionality=None - # grouping of parallel lines - if args['line_grouping']: - group_parallel_lines(network) - # parallisation if args['parallelisation']: parallelisation(network, start_snapshot=args['start_snapshot'], end_snapshot=args['end_snapshot'],group_size=1, solver_name=args['solver'], extra_functionality=extra_functionality) From 5b3ead29e5a12444c68b824c9bd7541d989d113c Mon Sep 17 00:00:00 2001 From: mariusves Date: Fri, 8 Sep 2017 11:08:57 +0200 Subject: [PATCH 58/65] small fix in sh_manipulation --- etrago/extras/utilities.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etrago/extras/utilities.py b/etrago/extras/utilities.py index e69e21fc..8466d6a3 100644 --- a/etrago/extras/utilities.py +++ b/etrago/extras/utilities.py @@ -155,7 +155,7 @@ def data_manipulation_sh (network): #trafo geom/topo network.transformers.set_value(new_trafo, 'geom', from_shape(MultiLineString([LineString([to_shape(network.buses.geom['25536']),point_bus1])]),4326)) - network.transformers.set_value(new_trafo, 'geom', from_shape(LineString([to_shape(network.buses.geom['25536']),point_bus1]),4326)) + network.transformers.set_value(new_trafo, 'topo', from_shape(LineString([to_shape(network.buses.geom['25536']),point_bus1]),4326)) return From f09777d886c224ba7b430f57394f2dc095da5454 Mon Sep 17 00:00:00 2001 From: mariusves Date: Fri, 8 Sep 2017 11:24:52 +0200 Subject: [PATCH 59/65] add ordering of buses --- etrago/extras/utilities.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/etrago/extras/utilities.py b/etrago/extras/utilities.py index 8466d6a3..6ac6f607 100644 --- a/etrago/extras/utilities.py +++ b/etrago/extras/utilities.py @@ -348,12 +348,22 @@ def normed(x): def group_parallel_lines(network): + #ordering of buses: (not sure if still necessary, remaining from SQL code) old_lines = network.lines + + for line in old_lines.index: + bus0_new = old_lines.loc[line,['bus0','bus1']].astype(int).min() + bus1_new = old_lines.loc[line,['bus0','bus1']].astype(int).max() + old_lines.set_value(line,'bus0',bus0_new) + old_lines.set_value(line,'bus1',bus1_new) + + # saving the old index for line in old_lines: old_lines['old_index'] = network.lines.index grouped = old_lines.groupby(['bus0','bus1']) + #calculating electrical properties for parallel lines grouped_agg = grouped.agg({ 'b': np.sum, 'b_pu': np.sum, 'cables': np.sum, @@ -390,7 +400,4 @@ def group_parallel_lines(network): new_lines=new_lines.drop('old_index',1) network.lines = new_lines - return - - - + return \ No newline at end of file From f18a4b4ca48645e204b0af20ff7fa8905b03fcaa Mon Sep 17 00:00:00 2001 From: mariusves Date: Fri, 8 Sep 2017 11:27:21 +0200 Subject: [PATCH 60/65] small hotfix for dtype of buses --- etrago/extras/utilities.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etrago/extras/utilities.py b/etrago/extras/utilities.py index 6ac6f607..2c37bd0f 100644 --- a/etrago/extras/utilities.py +++ b/etrago/extras/utilities.py @@ -352,8 +352,8 @@ def group_parallel_lines(network): old_lines = network.lines for line in old_lines.index: - bus0_new = old_lines.loc[line,['bus0','bus1']].astype(int).min() - bus1_new = old_lines.loc[line,['bus0','bus1']].astype(int).max() + bus0_new = str(old_lines.loc[line,['bus0','bus1']].astype(int).min()) + bus1_new = str(old_lines.loc[line,['bus0','bus1']].astype(int).max()) old_lines.set_value(line,'bus0',bus0_new) old_lines.set_value(line,'bus1',bus1_new) From 51f369b8757f87b5e4b4722eb3196c3036a7d975 Mon Sep 17 00:00:00 2001 From: Lukas Date: Fri, 8 Sep 2017 14:51:26 +0200 Subject: [PATCH 61/65] updated requirements for new release --- requirements.txt | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/requirements.txt b/requirements.txt index 045deff2..ed0d5c44 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,11 +1,10 @@ # use pip install -r requirements.txt to setup your virtualenv -# ego.powerflow release 0.0.4 -egopowerflow==0.0.4 +# ego.powerflow release 0.0.5 +-e git+https://github.com/openego/ego.powerflow.git@master#egg=ego.powerflow - -# ego.io release 0.2.0 -egoio==0.2.0 +# ego.io release 0.2.11 +-e git+https://github.com/openego/ego.io.git@master#egg=ego.io # eGo PyPSA fork on dev https://github.com/openego/PyPSA/tree/dev -e git+https://github.com/openego/PyPSA.git@dev#egg=PyPSA From 569c6e93404cfb61a5de60ca4e694a0869747444 Mon Sep 17 00:00:00 2001 From: Lukas Date: Fri, 8 Sep 2017 15:30:44 +0200 Subject: [PATCH 62/65] updated requirements for new releases --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index ed0d5c44..74e99cef 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,10 @@ # use pip install -r requirements.txt to setup your virtualenv # ego.powerflow release 0.0.5 --e git+https://github.com/openego/ego.powerflow.git@master#egg=ego.powerflow +egopowerflow==0.0.5 # ego.io release 0.2.11 --e git+https://github.com/openego/ego.io.git@master#egg=ego.io +egoio==0.2.11 # eGo PyPSA fork on dev https://github.com/openego/PyPSA/tree/dev -e git+https://github.com/openego/PyPSA.git@dev#egg=PyPSA From b0ac47c16ef934757b30cb5ecf55821bd0005e2d Mon Sep 17 00:00:00 2001 From: Lukas Date: Fri, 8 Sep 2017 15:52:23 +0200 Subject: [PATCH 63/65] updated setup.py to new versions --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index a2d41a20..f9a3245f 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ url='https://github.com/openego/eTraGo', license="GNU Affero General Public License Version 3 (AGPL-3.0)", packages=find_packages(), - install_requires=['egoio == 0.2.0', - 'egopowerflow == 0.0.4'], + install_requires=['egoio == 0.2.11', + 'egopowerflow == 0.0.5'], dependency_links=['git+ssh://git@github.com/openego/PyPSA.git@dev#egg=PyPSA'] ) From fba18119137f5dfe29c4e6768b50c1fca6505bb7 Mon Sep 17 00:00:00 2001 From: Lukas Date: Fri, 8 Sep 2017 15:54:45 +0200 Subject: [PATCH 64/65] updated version number --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f9a3245f..880233f3 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ author='NEXT ENERGY, ZNES Flensburg', author_email='', description='electrical Transmission Grid Optimization of flexibility options for transmission grids based on PyPSA', - version='0.1', + version='0.3', url='https://github.com/openego/eTraGo', license="GNU Affero General Public License Version 3 (AGPL-3.0)", packages=find_packages(), From 20a5ba3c3e75768f2b387c50aa8fa9220d763879 Mon Sep 17 00:00:00 2001 From: ulf Date: Fri, 8 Sep 2017 16:36:13 +0200 Subject: [PATCH 65/65] fix bug for SH NEP 2035 scenario --- etrago/appl.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/etrago/appl.py b/etrago/appl.py index 12bd03f3..823e1dc3 100644 --- a/etrago/appl.py +++ b/etrago/appl.py @@ -30,7 +30,7 @@ 'db': 'oedb', # db session 'gridversion':'v0.2.11', #None for model_draft or Version number (e.g. v0.2.10) for grid schema 'method': 'lopf', # lopf or pf - 'pf_post_lopf': True, #state whether you want to perform a pf after a lopf simulation + 'pf_post_lopf': False, #state whether you want to perform a pf after a lopf simulation 'start_snapshot': 2320, 'end_snapshot' : 2321, 'scn_name': 'SH NEP 2035', @@ -43,7 +43,7 @@ 'load_shedding':True, 'generator_noise':True, 'minimize_loading':False, - 'k_mean_clustering': True, + 'k_mean_clustering': False, 'parallelisation':False, 'comments': None} @@ -100,7 +100,7 @@ def etrago(args): (8760//(args['end_snapshot']-args['start_snapshot']+1))) # for SH scenario run do data preperation: - if args['scn_name'] == 'SH Status Quo': + if args['scn_name'] == 'SH Status Quo' or args['scn_name'] == 'SH NEP 2035': data_manipulation_sh(network) #load shedding in order to hunt infeasibilities