From 69ba3be08c8ed0327de990aa545617f2c3b06803 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20G=C3=BCnther?= Date: Thu, 9 Feb 2023 13:17:03 +0100 Subject: [PATCH 01/11] Run `black --preview` followed by just `black` On "src/egon/data/datasets/pypsaeursec/__init__.py" and with version `23.1.0`. Using `--preview` first followed by a normal invocation pulls in new style fixes but also makes sure, that only those remain, which will not conflict with normal `black` usage, e.g. when pre-commit hooks are run. This one removes some extraneous blank lines and wraps overly long ones. --- src/egon/data/datasets/pypsaeursec/__init__.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/egon/data/datasets/pypsaeursec/__init__.py b/src/egon/data/datasets/pypsaeursec/__init__.py index 84635631d..708484247 100755 --- a/src/egon/data/datasets/pypsaeursec/__init__.py +++ b/src/egon/data/datasets/pypsaeursec/__init__.py @@ -22,7 +22,6 @@ def run_pypsa_eur_sec(): - cwd = Path(".") filepath = cwd / "run-pypsa-eur-sec" filepath.mkdir(parents=True, exist_ok=True) @@ -130,7 +129,6 @@ def run_pypsa_eur_sec(): def read_network(): - # Set execute_pypsa_eur_sec to False until optional task is implemented execute_pypsa_eur_sec = False cwd = Path(".") @@ -264,7 +262,6 @@ def clean_database(): def neighbor_reduction(): - network = read_network() network.links.drop("pipe_retrofit", axis="columns", inplace=True) @@ -834,9 +831,15 @@ def links_to_etrago(neighbor_links, scn="eGon100RE", extendable=True): "ror": "run_of_river", "offwind-ac": "wind_offshore", "offwind-dc": "wind_offshore", - "urban_central_solar_thermal": "urban_central_solar_thermal_collector", - "residential_rural_solar_thermal": "residential_rural_solar_thermal_collector", - "services_rural_solar_thermal": "services_rural_solar_thermal_collector", + "urban_central_solar_thermal": ( + "urban_central_solar_thermal_collector" + ), + "residential_rural_solar_thermal": ( + "residential_rural_solar_thermal_collector" + ), + "services_rural_solar_thermal": ( + "services_rural_solar_thermal_collector" + ), }, inplace=True, ) From 1ca6e705e4fed2e75b9736bb1a7060d73562e63e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20G=C3=BCnther?= Date: Thu, 9 Feb 2023 15:21:12 +0100 Subject: [PATCH 02/11] Remove unused import --- src/egon/data/datasets/pypsaeursec/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/egon/data/datasets/pypsaeursec/__init__.py b/src/egon/data/datasets/pypsaeursec/__init__.py index 708484247..79e91a48f 100755 --- a/src/egon/data/datasets/pypsaeursec/__init__.py +++ b/src/egon/data/datasets/pypsaeursec/__init__.py @@ -17,7 +17,6 @@ from egon.data import __path__, db from egon.data.datasets import Dataset from egon.data.datasets.scenario_parameters import get_sector_parameters -import egon.data.config import egon.data.subprocess as subproc From 9ad2ca06ee0cb96e7ef2102d5848038bd1e1264b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20G=C3=BCnther?= Date: Thu, 9 Feb 2023 14:02:03 +0100 Subject: [PATCH 03/11] Wrap overly long line Take advantage of the fact that Python automatically joins adjacent strings by breaking the string at appropriate places to stay within the line length limit. Then wrap the two strings in parenthesis, making it one expression which can be spread over multiple lines. That way the strings can each be on individual lines and we're staying within 79 characters. --- src/egon/data/datasets/pypsaeursec/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/egon/data/datasets/pypsaeursec/__init__.py b/src/egon/data/datasets/pypsaeursec/__init__.py index 79e91a48f..4b20eb708 100755 --- a/src/egon/data/datasets/pypsaeursec/__init__.py +++ b/src/egon/data/datasets/pypsaeursec/__init__.py @@ -146,7 +146,10 @@ def read_network(): opts = data_config["scenario"]["opts"][0] sector_opts = data_config["scenario"]["sector_opts"][0] planning_horizons = data_config["scenario"]["planning_horizons"][0] - file = "elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc".format( + file = ( + "elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}" + "_{planning_horizons}.nc" + ).format( simpl=simpl, clusters=clusters, opts=opts, From 7f0b6aa3e2de6e2f0c42cc6208107cb705ba62e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20G=C3=BCnther?= Date: Thu, 9 Feb 2023 14:29:48 +0100 Subject: [PATCH 04/11] Use f-strings for fun and profit Actually, it's just to make the code shorter while also making it a bit more readable. (IMHO of course.) --- .../data/datasets/pypsaeursec/__init__.py | 45 +++++++------------ 1 file changed, 17 insertions(+), 28 deletions(-) diff --git a/src/egon/data/datasets/pypsaeursec/__init__.py b/src/egon/data/datasets/pypsaeursec/__init__.py index 4b20eb708..0b32e4a83 100755 --- a/src/egon/data/datasets/pypsaeursec/__init__.py +++ b/src/egon/data/datasets/pypsaeursec/__init__.py @@ -147,15 +147,8 @@ def read_network(): sector_opts = data_config["scenario"]["sector_opts"][0] planning_horizons = data_config["scenario"]["planning_horizons"][0] file = ( - "elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}" - "_{planning_horizons}.nc" - ).format( - simpl=simpl, - clusters=clusters, - opts=opts, - lv=lv, - sector_opts=sector_opts, - planning_horizons=planning_horizons, + f"elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}" + f"_{planning_horizons}.nc" ) target_file = ( @@ -202,9 +195,9 @@ def clean_database(): for comp in comp_one_port: db.execute_sql( f""" - DELETE FROM {"grid.egon_etrago_" + comp + "_timeseries"} - WHERE {comp + "_id"} IN ( - SELECT {comp + "_id"} FROM {"grid.egon_etrago_" + comp} + DELETE FROM grid.egon_etrago_{comp}_timeseries + WHERE {comp}_id IN ( + SELECT {comp}_id FROM grid.egon_etrago_{comp} WHERE bus IN ( SELECT bus_id FROM grid.egon_etrago_bus WHERE country != 'DE' @@ -212,7 +205,7 @@ def clean_database(): AND scn_name = '{scn_name}' ); - DELETE FROM {"grid.egon_etrago_" + comp} + DELETE FROM grid.egon_etrago_{comp} WHERE bus IN ( SELECT bus_id FROM grid.egon_etrago_bus WHERE country != 'DE' @@ -229,10 +222,10 @@ def clean_database(): for comp, id in zip(comp_2_ports, ["line_id", "trafo_id", "link_id"]): db.execute_sql( f""" - DELETE FROM {"grid.egon_etrago_" + comp + "_timeseries"} + DELETE FROM grid.egon_etrago_{comp}_timeseries WHERE scn_name = '{scn_name}' AND {id} IN ( - SELECT {id} FROM {"grid.egon_etrago_" + comp} + SELECT {id} FROM grid.egon_etrago_{comp} WHERE "bus0" IN ( SELECT bus_id FROM grid.egon_etrago_bus WHERE country != 'DE' @@ -243,7 +236,7 @@ def clean_database(): AND scn_name = '{scn_name}') ); - DELETE FROM {"grid.egon_etrago_" + comp} + DELETE FROM grid.egon_etrago_{comp} WHERE scn_name = '{scn_name}' AND "bus0" IN ( SELECT bus_id FROM grid.egon_etrago_bus @@ -308,13 +301,11 @@ def neighbor_reduction(): for i, k in zip(lines_cb_1.bus1.values, lines_cb_1.index): network.add( "Load", - "slack_fix " + i + " " + k, + f"slack_fix {i} {k}", bus=i, p_set=network.lines_t.p1[k], ) - network.loads.carrier.loc[ - "slack_fix " + i + " " + k - ] = lines_cb_1.carrier[k] + network.loads.carrier.loc[f"slack_fix {i} {k}"] = lines_cb_1.carrier[k] # select all lines which have at bus0 the bus which is kept lines_cb_0 = network.lines[ @@ -325,13 +316,11 @@ def neighbor_reduction(): for i, k in zip(lines_cb_0.bus0.values, lines_cb_0.index): network.add( "Load", - "slack_fix " + i + " " + k, + f"slack_fix {i} {k}", bus=i, p_set=network.lines_t.p0[k], ) - network.loads.carrier.loc[ - "slack_fix " + i + " " + k - ] = lines_cb_0.carrier[k] + network.loads.carrier.loc[f"slack_fix {i} {k}"] = lines_cb_0.carrier[k] # do the same for links @@ -351,12 +340,12 @@ def neighbor_reduction(): for i, k in zip(links_cb_1.bus1.values, links_cb_1.index): network.add( "Load", - "slack_fix_links " + i + " " + k, + f"slack_fix_links {i} {k}", bus=i, p_set=network.links_t.p1[k], ) network.loads.carrier.loc[ - "slack_fix_links " + i + " " + k + f"slack_fix_links {i} {k}" ] = links_cb_1.carrier[k] # select all links which have at bus0 the bus which is kept @@ -368,12 +357,12 @@ def neighbor_reduction(): for i, k in zip(links_cb_0.bus0.values, links_cb_0.index): network.add( "Load", - "slack_fix_links " + i + " " + k, + f"slack_fix_links {i} {k}", bus=i, p_set=network.links_t.p0[k], ) network.loads.carrier.loc[ - "slack_fix_links " + i + " " + k + f"slack_fix_links {i} {k}" ] = links_cb_0.carrier[k] # drop remaining foreign components From ddd224fc8cfdd2b73fe8734507303045922f1733 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20G=C3=BCnther?= Date: Wed, 8 Feb 2023 17:20:45 +0100 Subject: [PATCH 05/11] Use `~` instead of `== False` to negate `Series` Using `== False` can not only be unnecessarily verbose, but also surprisingly incorrect, as `0 == False` is `True`. Since `pandas` doesn't want us to use `not` with boolean `Series`, `~` is the correct choice for flipping those `Series`. This also squelches `flake8` complaints. --- .../data/datasets/pypsaeursec/__init__.py | 36 +++++++++---------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/src/egon/data/datasets/pypsaeursec/__init__.py b/src/egon/data/datasets/pypsaeursec/__init__.py index 0b32e4a83..1eeae3283 100755 --- a/src/egon/data/datasets/pypsaeursec/__init__.py +++ b/src/egon/data/datasets/pypsaeursec/__init__.py @@ -287,14 +287,14 @@ def neighbor_reduction(): network.lines = network.lines.drop( network.lines[ - (network.lines["bus0"].isin(network.buses.index) == False) - & (network.lines["bus1"].isin(network.buses.index) == False) + ~network.lines.loc[:, "bus0"].isin(network.buses.index) + & ~network.lines["bus1"].isin(network.buses.index) ].index ) # select all lines which have at bus1 the bus which is kept lines_cb_1 = network.lines[ - (network.lines["bus0"].isin(network.buses.index) == False) + ~network.lines["bus0"].isin(network.buses.index) ] # create a load at bus1 with the line's hourly loading @@ -309,7 +309,7 @@ def neighbor_reduction(): # select all lines which have at bus0 the bus which is kept lines_cb_0 = network.lines[ - (network.lines["bus1"].isin(network.buses.index) == False) + ~network.lines["bus1"].isin(network.buses.index) ] # create a load at bus0 with the line's hourly loading @@ -326,14 +326,14 @@ def neighbor_reduction(): network.links = network.links.drop( network.links[ - (network.links["bus0"].isin(network.buses.index) == False) - & (network.links["bus1"].isin(network.buses.index) == False) + ~network.links["bus0"].isin(network.buses.index) + & ~network.links["bus1"].isin(network.buses.index) ].index ) # select all links which have at bus1 the bus which is kept links_cb_1 = network.links[ - (network.links["bus0"].isin(network.buses.index) == False) + ~network.links["bus0"].isin(network.buses.index) ] # create a load at bus1 with the link's hourly loading @@ -350,7 +350,7 @@ def neighbor_reduction(): # select all links which have at bus0 the bus which is kept links_cb_0 = network.links[ - (network.links["bus1"].isin(network.buses.index) == False) + ~network.links["bus1"].isin(network.buses.index) ] # create a load at bus0 with the link's hourly loading @@ -369,39 +369,37 @@ def neighbor_reduction(): network.lines = network.lines.drop( network.lines[ - (network.lines["bus0"].isin(network.buses.index) == False) - | (network.lines["bus1"].isin(network.buses.index) == False) + ~network.lines["bus0"].isin(network.buses.index) + | ~network.lines["bus1"].isin(network.buses.index) ].index ) network.links = network.links.drop( network.links[ - (network.links["bus0"].isin(network.buses.index) == False) - | (network.links["bus1"].isin(network.buses.index) == False) + ~network.links["bus0"].isin(network.buses.index) + | ~network.links["bus1"].isin(network.buses.index) ].index ) network.transformers = network.transformers.drop( network.transformers[ - (network.transformers["bus0"].isin(network.buses.index) == False) - | (network.transformers["bus1"].isin(network.buses.index) == False) + ~network.transformers["bus0"].isin(network.buses.index) + | ~network.transformers["bus1"].isin(network.buses.index) ].index ) network.generators = network.generators.drop( network.generators[ - (network.generators["bus"].isin(network.buses.index) == False) + ~network.generators["bus"].isin(network.buses.index) ].index ) network.loads = network.loads.drop( - network.loads[ - (network.loads["bus"].isin(network.buses.index) == False) - ].index + network.loads[~network.loads["bus"].isin(network.buses.index)].index ) network.storage_units = network.storage_units.drop( network.storage_units[ - (network.storage_units["bus"].isin(network.buses.index) == False) + ~network.storage_units["bus"].isin(network.buses.index) ].index ) From e7c601ee65bfa327184c4e98fbc28e5161d3df39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20G=C3=BCnther?= Date: Thu, 9 Feb 2023 01:19:15 +0100 Subject: [PATCH 06/11] Use descriptive variable names Replace single and dual letter variable names with descriptive ones. Remove comments as the variable names now serve a purpose comparable to that of the comments. Feel free to reintroduce the comments if you feel that they serve an additional informatory purpose. But please expand them a bit, at least noting that they are examples of what's happening in specific exemplary loop iterations. That is, if I understood them correctly. If I'm wrong, please expand them to prevent that misunderstanding. --- .../data/datasets/pypsaeursec/__init__.py | 51 ++++++++++--------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/src/egon/data/datasets/pypsaeursec/__init__.py b/src/egon/data/datasets/pypsaeursec/__init__.py index 1eeae3283..6d987698b 100755 --- a/src/egon/data/datasets/pypsaeursec/__init__.py +++ b/src/egon/data/datasets/pypsaeursec/__init__.py @@ -411,17 +411,16 @@ def neighbor_reduction(): "transformers", "links", ] - for g in components: # loads_t - h = g + "_t" - nw = getattr(network, h) # network.loads_t - for i in nw.keys(): # network.loads_t.p - cols = [ - j - for j in getattr(nw, i).columns - if j not in getattr(network, g).index + for component in components: + component_t = getattr(network, f"{component}_t") + for key in component_t.keys(): + columns = [ + column + for column in getattr(component_t, key).columns + if column not in getattr(network, component).index ] - for k in cols: - del getattr(nw, i)[k] + for column in columns: + del getattr(component_t, key)[column] # writing components of neighboring countries to etrago tables @@ -453,9 +452,11 @@ def neighbor_reduction(): neighbor_lines.index += db.next_etrago_id("line") if not network.lines_t["s_max_pu"].empty: - for i in neighbor_lines_t.columns: - new_index = neighbor_lines[neighbor_lines["name"] == i].index - neighbor_lines_t.rename(columns={i: new_index[0]}, inplace=True) + for column in neighbor_lines_t.columns: + new_index = neighbor_lines[neighbor_lines["name"] == column].index + neighbor_lines_t.rename( + columns={column: new_index[0]}, inplace=True + ) # links neighbor_links = network.links[ @@ -488,9 +489,9 @@ def neighbor_reduction(): ) neighbor_gens.index += db.next_etrago_id("generator") - for i in neighbor_gens_t.columns: - new_index = neighbor_gens[neighbor_gens["name"] == i].index - neighbor_gens_t.rename(columns={i: new_index[0]}, inplace=True) + for column in neighbor_gens_t.columns: + new_index = neighbor_gens[neighbor_gens["name"] == column].index + neighbor_gens_t.rename(columns={column: new_index[0]}, inplace=True) # loads @@ -506,9 +507,9 @@ def neighbor_reduction(): ) neighbor_loads.index += db.next_etrago_id("load") - for i in neighbor_loads_t.columns: - new_index = neighbor_loads[neighbor_loads["index"] == i].index - neighbor_loads_t.rename(columns={i: new_index[0]}, inplace=True) + for column in neighbor_loads_t.columns: + new_index = neighbor_loads[neighbor_loads["index"] == column].index + neighbor_loads_t.rename(columns={column: new_index[0]}, inplace=True) # stores neighbor_stores = network.stores[network.stores.bus.isin(neighbors.index)] @@ -523,9 +524,9 @@ def neighbor_reduction(): ) neighbor_stores.index += db.next_etrago_id("store") - for i in neighbor_stores_t.columns: - new_index = neighbor_stores[neighbor_stores["name"] == i].index - neighbor_stores_t.rename(columns={i: new_index[0]}, inplace=True) + for column in neighbor_stores_t.columns: + new_index = neighbor_stores[neighbor_stores["name"] == column].index + neighbor_stores_t.rename(columns={column: new_index[0]}, inplace=True) # storage_units neighbor_storage = network.storage_units[ @@ -546,9 +547,9 @@ def neighbor_reduction(): ) neighbor_storage.index += db.next_etrago_id("storage") - for i in neighbor_storage_t.columns: - new_index = neighbor_storage[neighbor_storage["name"] == i].index - neighbor_storage_t.rename(columns={i: new_index[0]}, inplace=True) + for column in neighbor_storage_t.columns: + new_index = neighbor_storage[neighbor_storage["name"] == column].index + neighbor_storage_t.rename(columns={column: new_index[0]}, inplace=True) # Connect to local database engine = db.engine() From 54ac272e62515409b5a8b652b4fb2b6ee28e9ee4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20G=C3=BCnther?= Date: Thu, 16 Feb 2023 15:09:43 +0100 Subject: [PATCH 07/11] Replace `getattr` with key lookup The `component_t` object is an instance of `pypsa.descriptors.Dict`, which allows both, attribute access and key lookup. Therefore it's not necessary to use `getattr`. --- src/egon/data/datasets/pypsaeursec/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/egon/data/datasets/pypsaeursec/__init__.py b/src/egon/data/datasets/pypsaeursec/__init__.py index 6d987698b..4fc16a09b 100755 --- a/src/egon/data/datasets/pypsaeursec/__init__.py +++ b/src/egon/data/datasets/pypsaeursec/__init__.py @@ -416,11 +416,11 @@ def neighbor_reduction(): for key in component_t.keys(): columns = [ column - for column in getattr(component_t, key).columns + for column in component_t[key].columns if column not in getattr(network, component).index ] for column in columns: - del getattr(component_t, key)[column] + del component_t[key][column] # writing components of neighboring countries to etrago tables From 87d679abe446180385449fc33d021955de277409 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20G=C3=BCnther?= Date: Thu, 9 Feb 2023 01:21:20 +0100 Subject: [PATCH 08/11] Try to make code easier to understand The usage of `i` to store `DataFrame` column names was very confusing to me. It correctly suggested (to me) that `i` is an integer index so I got that `x["foo"][i]` is accessing column `"foo"` of the `DataFrame` stored in `x` and then pulling out the value at index `i` of the resulting `Series`. But since I now associated `i` with being an integer index, the left hand side of `x["foo"][i] = y[i].values.tolist()`, i.e. `y[i].values.tolist()`, didn't make sense to me, because `y[i]` looks like it's pulling out the single value of a `Series` or list at an integer index. But it doesn't. It actually pulls the column `i` out of the `DataFrame` `y`. That can then be converted to a list and is assigned to the `DataFrame` cell `x["foo"][i]`. Figuring this out took me hours, so I decided to change the variable name to `column` to convey that it's meant to store a column, even if it is sometimes used to index a row. What is indexed is then made explicit by using `.at` and `.loc` as appropriate. Using `.at[row, column]` also serves the purpose of squelching some instances of ``` SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-doc# ``` Using `.at[r, c]` is necessary in order to tell `pandas` that we want to assign a sequence to a single `DataFrame` location. Using `.loc[r, c]` `pandas` would try to somehow align the sequence with the cell's neighbourhood which usually results in an error. --- .../data/datasets/pypsaeursec/__init__.py | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/src/egon/data/datasets/pypsaeursec/__init__.py b/src/egon/data/datasets/pypsaeursec/__init__.py index 4fc16a09b..1da052015 100755 --- a/src/egon/data/datasets/pypsaeursec/__init__.py +++ b/src/egon/data/datasets/pypsaeursec/__init__.py @@ -948,9 +948,9 @@ def links_to_etrago(neighbor_links, scn="eGon100RE", extendable=True): ) neighbor_loads_t_etrago["scn_name"] = "eGon100RE" neighbor_loads_t_etrago["temp_id"] = 1 - for i in neighbor_loads_t.columns: - neighbor_loads_t_etrago["p_set"][i] = neighbor_loads_t[ - i + for column in neighbor_loads_t.columns: + neighbor_loads_t_etrago.at[column, "p_set"] = neighbor_loads_t.loc[ + :, column ].values.tolist() neighbor_loads_t_etrago.to_sql( @@ -969,9 +969,9 @@ def links_to_etrago(neighbor_links, scn="eGon100RE", extendable=True): ) neighbor_gens_t_etrago["scn_name"] = "eGon100RE" neighbor_gens_t_etrago["temp_id"] = 1 - for i in neighbor_gens_t.columns: - neighbor_gens_t_etrago["p_max_pu"][i] = neighbor_gens_t[ - i + for column in neighbor_gens_t.columns: + neighbor_gens_t_etrago.at[column, "p_max_pu"] = neighbor_gens_t.loc[ + :, column ].values.tolist() neighbor_gens_t_etrago.to_sql( @@ -990,10 +990,10 @@ def links_to_etrago(neighbor_links, scn="eGon100RE", extendable=True): ) neighbor_stores_t_etrago["scn_name"] = "eGon100RE" neighbor_stores_t_etrago["temp_id"] = 1 - for i in neighbor_stores_t.columns: - neighbor_stores_t_etrago["e_min_pu"][i] = neighbor_stores_t[ - i - ].values.tolist() + for column in neighbor_stores_t.columns: + neighbor_stores_t_etrago.at[ + column, "e_min_pu" + ] = neighbor_stores_t.loc[:, column].values.tolist() neighbor_stores_t_etrago.to_sql( "egon_etrago_store_timeseries", @@ -1011,10 +1011,10 @@ def links_to_etrago(neighbor_links, scn="eGon100RE", extendable=True): ) neighbor_storage_t_etrago["scn_name"] = "eGon100RE" neighbor_storage_t_etrago["temp_id"] = 1 - for i in neighbor_storage_t.columns: - neighbor_storage_t_etrago["inflow"][i] = neighbor_storage_t[ - i - ].values.tolist() + for column in neighbor_storage_t.columns: + neighbor_storage_t_etrago.at[ + column, "inflow" + ] = neighbor_storage_t.loc[:, column].values.tolist() neighbor_storage_t_etrago.to_sql( "egon_etrago_storage_timeseries", @@ -1032,10 +1032,10 @@ def links_to_etrago(neighbor_links, scn="eGon100RE", extendable=True): ) neighbor_lines_t_etrago["scn_name"] = "eGon100RE" - for i in neighbor_lines_t.columns: - neighbor_lines_t_etrago["s_max_pu"][i] = neighbor_lines_t[ - i - ].values.tolist() + for column in neighbor_lines_t.columns: + neighbor_lines_t_etrago.at[ + column, "s_max_pu" + ] = neighbor_lines_t.loc[:, column].values.tolist() neighbor_lines_t_etrago.to_sql( "egon_etrago_line_timeseries", From 70298a1db6b729e179f5f083920f1cfb7228611c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20G=C3=BCnther?= Date: Wed, 8 Feb 2023 20:26:47 +0100 Subject: [PATCH 09/11] Use `.loc[:, c]` to assign to `DataFrame` columns Instead of `[c]`. This hopefully squelches a few instances of ``` SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-doc# ``` It should also make sure that the assignment always happens and isn't lost in an ephemeral copy. --- .../data/datasets/pypsaeursec/__init__.py | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/egon/data/datasets/pypsaeursec/__init__.py b/src/egon/data/datasets/pypsaeursec/__init__.py index 1da052015..9514aa064 100755 --- a/src/egon/data/datasets/pypsaeursec/__init__.py +++ b/src/egon/data/datasets/pypsaeursec/__init__.py @@ -428,7 +428,7 @@ def neighbor_reduction(): network.buses.country = network.buses.index.str[:2] neighbors = network.buses[network.buses.country != "DE"] - neighbors["new_index"] = ( + neighbors.loc[:, "new_index"] = ( db.next_etrago_id("bus") + neighbors.reset_index().index ) @@ -554,7 +554,7 @@ def neighbor_reduction(): # Connect to local database engine = db.engine() - neighbors["scn_name"] = "eGon100RE" + neighbors.loc[:, "scn_name"] = "eGon100RE" neighbors.index = neighbors["new_index"] # Correct geometry for non AC buses @@ -609,11 +609,11 @@ def neighbor_reduction(): # prepare and write neighboring crossborder lines to etrago tables def lines_to_etrago(neighbor_lines=neighbor_lines, scn="eGon100RE"): - neighbor_lines["scn_name"] = scn - neighbor_lines["cables"] = 3 * neighbor_lines["num_parallel"].astype( - int - ) - neighbor_lines["s_nom"] = neighbor_lines["s_nom_min"] + neighbor_lines.loc[:, "scn_name"] = scn + neighbor_lines.loc[:, "cables"] = 3 * neighbor_lines[ + "num_parallel" + ].astype(int) + neighbor_lines.loc[:, "s_nom"] = neighbor_lines["s_nom_min"] for i in [ "name", @@ -689,7 +689,7 @@ def links_to_etrago(neighbor_links, scn="eGon100RE", extendable=True): None """ - neighbor_links["scn_name"] = scn + neighbor_links.loc[:, "scn_name"] = scn if extendable is True: neighbor_links = neighbor_links.drop( @@ -808,9 +808,9 @@ def links_to_etrago(neighbor_links, scn="eGon100RE", extendable=True): links_to_etrago(neighbor_links[neighbor_links.carrier == "DC"], "eGon2035") # prepare neighboring generators for etrago tables - neighbor_gens["scn_name"] = "eGon100RE" - neighbor_gens["p_nom"] = neighbor_gens["p_nom_opt"] - neighbor_gens["p_nom_extendable"] = False + neighbor_gens.loc[:, "scn_name"] = "eGon100RE" + neighbor_gens.loc[:, "p_nom"] = neighbor_gens["p_nom_opt"] + neighbor_gens.loc[:, "p_nom_extendable"] = False # Unify carrier names neighbor_gens.carrier = neighbor_gens.carrier.str.replace(" ", "_") @@ -847,7 +847,7 @@ def links_to_etrago(neighbor_links, scn="eGon100RE", extendable=True): ) # prepare neighboring loads for etrago tables - neighbor_loads["scn_name"] = "eGon100RE" + neighbor_loads.loc[:, "scn_name"] = "eGon100RE" # Unify carrier names neighbor_loads.carrier = neighbor_loads.carrier.str.replace(" ", "_") @@ -879,7 +879,7 @@ def links_to_etrago(neighbor_links, scn="eGon100RE", extendable=True): ) # prepare neighboring stores for etrago tables - neighbor_stores["scn_name"] = "eGon100RE" + neighbor_stores.loc[:, "scn_name"] = "eGon100RE" # Unify carrier names neighbor_stores.carrier = neighbor_stores.carrier.str.replace(" ", "_") @@ -919,7 +919,7 @@ def links_to_etrago(neighbor_links, scn="eGon100RE", extendable=True): ) # prepare neighboring storage_units for etrago tables - neighbor_storage["scn_name"] = "eGon100RE" + neighbor_storage.loc[:, "scn_name"] = "eGon100RE" # Unify carrier names neighbor_storage.carrier = neighbor_storage.carrier.str.replace(" ", "_") From e908207d9b74d44ef8a5ef9fed6a7a885db55c17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20G=C3=BCnther?= Date: Thu, 9 Feb 2023 01:08:34 +0100 Subject: [PATCH 10/11] Replace `[column]` access with `.loc[:, column]` At least in the places where I noticed. This makes it easier to see, whether the indexing is done into a dictionary, `Series` or list, i.e. a normally one dimensional structure, or a `DataFrame`, which is always two dimensional. --- src/egon/data/datasets/pypsaeursec/__init__.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/egon/data/datasets/pypsaeursec/__init__.py b/src/egon/data/datasets/pypsaeursec/__init__.py index 9514aa064..95785fb61 100755 --- a/src/egon/data/datasets/pypsaeursec/__init__.py +++ b/src/egon/data/datasets/pypsaeursec/__init__.py @@ -555,7 +555,7 @@ def neighbor_reduction(): engine = db.engine() neighbors.loc[:, "scn_name"] = "eGon100RE" - neighbors.index = neighbors["new_index"] + neighbors.index = neighbors.loc[:, "new_index"] # Correct geometry for non AC buses carriers = set(neighbors.carrier.to_list()) @@ -610,10 +610,10 @@ def neighbor_reduction(): # prepare and write neighboring crossborder lines to etrago tables def lines_to_etrago(neighbor_lines=neighbor_lines, scn="eGon100RE"): neighbor_lines.loc[:, "scn_name"] = scn - neighbor_lines.loc[:, "cables"] = 3 * neighbor_lines[ - "num_parallel" + neighbor_lines.loc[:, "cables"] = 3 * neighbor_lines.loc[ + :, "num_parallel" ].astype(int) - neighbor_lines.loc[:, "s_nom"] = neighbor_lines["s_nom_min"] + neighbor_lines.loc[:, "s_nom"] = neighbor_lines.loc[:, "s_nom_min"] for i in [ "name", @@ -809,7 +809,7 @@ def links_to_etrago(neighbor_links, scn="eGon100RE", extendable=True): # prepare neighboring generators for etrago tables neighbor_gens.loc[:, "scn_name"] = "eGon100RE" - neighbor_gens.loc[:, "p_nom"] = neighbor_gens["p_nom_opt"] + neighbor_gens.loc[:, "p_nom"] = neighbor_gens.loc[:, "p_nom_opt"] neighbor_gens.loc[:, "p_nom_extendable"] = False # Unify carrier names From 59e17b7a36a9f11b1ba4b8d8b0bde453f81aca9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20G=C3=BCnther?= Date: Sat, 18 Feb 2023 13:01:02 +0100 Subject: [PATCH 11/11] Bump `PypsaEurSec`'s dataset version --- src/egon/data/datasets/pypsaeursec/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/egon/data/datasets/pypsaeursec/__init__.py b/src/egon/data/datasets/pypsaeursec/__init__.py index 95785fb61..7a28a7f08 100755 --- a/src/egon/data/datasets/pypsaeursec/__init__.py +++ b/src/egon/data/datasets/pypsaeursec/__init__.py @@ -1060,7 +1060,7 @@ class PypsaEurSec(Dataset): def __init__(self, dependencies): super().__init__( name="PypsaEurSec", - version="0.0.9", + version="0.0.10", dependencies=dependencies, tasks=tasks, )