Changes made:

*git-ignore file has been updated to avoid all .geojson files
*details of new parameters have been added to the configtables (clean_osm_data_options.csv)
*the config.default and config.tutorial files have been renamed and comments have been formated to fit PEP 8 recommendations
*clean_osm_data script has been renamed and now the description of the added function follows PEP8 format and matches other functions
*the proposed function in clean_osm_data has now been tested for substations also and is now being used in substations and cables

.gitignore

@@ -23,7 +23,7 @@ dag.svg
 # Files appear for tests
 configs/scenarios/config.custom.yaml
 data/*.csv
-data/custom_lines.geojson
+data/*.geojson
 
 # Untracked folders (and files within)
 img/

config.default.yaml

@@ -101,12 +101,12 @@ clean_osm_data_options:  # osm = OpenStreetMap
   tag_substation: "transmission"  # Filters only substations with 'transmission' tag, ('distribution' also available)
   add_line_endings: true  # When "True", then line endings are added to the dataset of the substations
   generator_name_method: OSM  # Methodology to specify the name to the generator. Options: OSM (name as by OSM dataset), closest_city (name by the closest city)
-  use_custom_lines: "OSM_only"  # Flag added to use customized (Custom_only), OSM-downloaded (OSM_only), or both (Add_custom) in the clean_osm_data script (custom information should follow the format of the respective OSM .geojson file in the data folder)
-  custom_path_lines: False # Provide the specific absolute path of the custom file including its name and type e.g. (C:\...\...\pypsa-earth\data\custom_lines.geojson)
-  use_custom_substations: "OSM_only"  # Flag added to use customized (Custom_only), OSM-downloaded (OSM_only), or both (Add_custom) in the clean_osm_data script (custom information should follow the format of the respective OSM .geojson file in the data folder)
-  custom_path_cables: False # Provide the specific absolute path of the custom file including its name and type e.g. (C:\...\...\pypsa-earth\data\custom_substations.geojson)
-  use_custom_substations: "OSM_only"  # Flag added to use customized (Custom_only), OSM-downloaded (OSM_only), or both (Add_custom) in the clean_osm_data script (custom information should follow the format of the respective OSM .geojson file in the data folder)
-  custom_path_cables: False # Provide the specific absolute path of the custom file including its name and type e.g. (C:\...\...\pypsa-earth\data\custom_cables.geojson)
+  use_custom_lines: "OSM_only"  # Use OSM (OSM_only), customized (Custom_only), or both data sets (Add_custom)
+  path_custom_lines: False  # If exists, provide the specific absolute path of the custom file e.g. (C:\...\...\pypsa-earth\data\custom_lines.geojson)
+  use_custom_substations: "OSM_only"  # Use OSM (OSM_only), customized (Custom_only), or both data sets (Add_custom)
+  path_custom_substations: False  # If exists, provide the specific absolute path of the custom file e.g. (C:\...\...\pypsa-earth\data\custom_substations.geojson)
+  use_custom_cables: "OSM_only"  # Use OSM (OSM_only), customized (Custom_only), or both data sets (Add_custom)
+  path_custom_cables: False  # If exists, provide the specific absolute path of the custom file e.g. (C:\...\...\pypsa-earth\data\custom_cables.geojson)
 
 build_osm_network:  # Options of the build_osm_network script; osm = OpenStreetMap
   group_close_buses: true  # When "True", close buses are merged and guarantee the voltage matching among line endings

config.tutorial.yaml

@@ -115,12 +115,12 @@ clean_osm_data_options:
   tag_substation: "transmission"  # needed feature tag to be considered for the analysis. If empty, no filtering on the tag_substation is performed
   add_line_endings: true  # When true, the line endings are added to the dataset of the substations
   generator_name_method: OSM  # Methodology to specify the name to the generator. Options: OSM (name as by OSM dataset), closest_city (name by the closest city)
-  use_custom_lines: "OSM_only"  # Flag added to use customized (Custom_only), OSM-downloaded (OSM_only), or both (Add_custom) in the clean_osm_data script (custom information should follow the format of the respective OSM .geojson file in the data folder)
-  custom_path_lines: false # Provide the specific absolute path of the custom file including its name and type e.g. (C:\...\...\pypsa-earth\data\custom_lines.geojson)
-  use_custom_substations: "OSM_only"  # Flag added to use customized (Custom_only), OSM-downloaded (OSM_only), or both (Add_custom) in the clean_osm_data script (custom information should follow the format of the respective OSM .geojson file in the data folder)
-  custom_path_substations: false # Provide the specific absolute path of the custom file including its name and type e.g. (C:\...\...\pypsa-earth\data\custom_substations.geojson)
-  use_custom_cables: "OSM_only"  # Flag added to use customized (Custom_only), OSM-downloaded (OSM_only), or both (Add_custom) in the clean_osm_data script (custom information should follow the format of the respective OSM .geojson file in the data folder)
-  custom_path_cables: false # Provide the specific absolute path of the custom file including its name and type e.g. (C:\...\...\pypsa-earth\data\custom_cables.geojson)
+  use_custom_lines: "OSM_only"  # Use OSM (OSM_only), customized (Custom_only), or both data sets (Add_custom)
+  path_custom_lines: False  # If exists, provide the specific absolute path of the custom file e.g. (C:\...\...\pypsa-earth\data\custom_lines.geojson)
+  use_custom_substations: "OSM_only"  # Use OSM (OSM_only), customized (Custom_only), or both data sets (Add_custom)
+  path_custom_substations: False  # If exists, provide the specific absolute path of the custom file e.g. (C:\...\...\pypsa-earth\data\custom_substations.geojson)
+  use_custom_cables: "OSM_only"  # Use OSM (OSM_only), customized (Custom_only), or both data sets (Add_custom)
+  path_custom_cables: False  # If exists, provide the specific absolute path of the custom file e.g. (C:\...\...\pypsa-earth\data\custom_cables.geojson)
 
 build_osm_network:  # Options of the build_osm_network script; osm = OpenStreetMap
   group_close_buses: true  # When "True", close buses are merged and guarantee the voltage matching among line endings

doc/configtables/clean_osm_data_options.csv

@@ -4,3 +4,6 @@ threshold_voltage, V,, "Assets below the voltage threshold will not be considere
 tag_substation,, "{""transmission"", ""distribution""}", "Filters only substations with the corresponding tag (""transmission"" or ""distribution"")."
 add_line_endings, bool, "{True, False}", "True: line endings are added to the dataset of the substations."
 generator_name_method,, "{OSM, closest_city}", "Methodology to specify the name of generators. From OpenStreetMap (OSM) or named after the closest city (closest_city)."
+use_custom_lines,,"{OSM_only, Custom_only, Add_custom}", "Method for selecting which type of data is used for lines in the clean_osm_data script. Use OSM (OSM_only), customized (Custom_only), or both data sets (Add_custom)."
+use_custom_cables,,"{OSM_only, Custom_only, Add_custom}", "Method for selecting which type of data is used for cables in the clean_osm_data script. Use OSM (OSM_only), customized (Custom_only), or both data sets (Add_custom)."
+use_custom_substations,,"{OSM_only, Custom_only, Add_custom}", "Method for selecting which type of data is used for substations in the clean_osm_data script. Use OSM (OSM_only), customized (Custom_only), or both data sets (Add_custom)."

scripts/clean_osm_data.py

@@ -835,33 +835,24 @@ def set_name_by_closestcity(df_all_generators, colname="name"):
     return df_all_generators
 
 
-def use_custom_data_files(
-    custom_component_type,
-):  #### custom_data_type should be a string named lines, cables or substations
-    # checks the type of data/components to be reviewed
-    if custom_component_type == "lines":
-        custom_conditional = snakemake.config["clean_osm_data_options"][
-            "use_custom_lines"
-        ]
-        custom_path = snakemake.config["clean_osm_data_options"].get(
-            "custom_path_lines", False
-        )
-
-    elif custom_component_type == "substations":
-        custom_conditional = snakemake.config["clean_osm_data_options"][
-            "use_custom_substations"
-        ]
-        custom_path = snakemake.config["clean_osm_data_options"].get(
-            "custom_path_substations", False
-        )
-
-    elif custom_component_type == "cables":
-        custom_conditional = snakemake.config["clean_osm_data_options"][
-            "use_custom_cables"
-        ]
-        custom_path = snakemake.config["clean_osm_data_options"].get(
-            "custom_path_cables", False
-        )
+def use_custom_data_files (custom_component_type):
+    """
+    Function to check if OSM or custom data should be considered. 
+    the custom_component_type should be a string named lines, cables or substations
+    """       
+
+    #checks the type of data/components to be reviewed
+    if custom_component_type == 'lines' :
+        custom_conditional = snakemake.config["clean_osm_data_options"]["use_custom_lines"]
+        custom_path = snakemake.config["clean_osm_data_options"].get("path_custom_lines",False)
+
+    elif custom_component_type == 'substations' :
+        custom_conditional = snakemake.config["clean_osm_data_options"]["use_custom_substations"]
+        custom_path = snakemake.config["clean_osm_data_options"].get("path_custom_substations",False)
+
+    elif custom_component_type == 'cables' :
+        custom_conditional = snakemake.config["clean_osm_data_options"]["use_custom_cables"]
+        custom_path = snakemake.config["clean_osm_data_options"].get("path_custom_cables",False)
 
     else:
         raise ValueError(
@@ -922,7 +913,7 @@ def clean_data(
     if os.path.getsize(input_files["cables"]) > 0:
         logger.info("Add OSM cables to data")
         # Load raw data lines
-        df_cables = gpd.read_file(input_files["cables"])
+        df_cables = use_custom_data_files ('cables')
 
         # prepare cables dataframe and data types
         df_cables = prepare_lines_df(df_cables)
@@ -970,7 +961,7 @@ def clean_data(
     logger.info("Process OSM substations")
 
     if os.path.getsize(input_files["substations"]) > 0:
-        df_all_substations = gpd.read_file(input_files["substations"])
+        df_all_substations = use_custom_data_files ('substations')
 
         # prepare dataset for substations
         df_all_substations = prepare_substation_df(df_all_substations)