From 9566c0dafa15c943262cdcc70f4b36a9622dd353 Mon Sep 17 00:00:00 2001 From: carlosfv Date: Tue, 26 Sep 2023 12:24:16 -0400 Subject: [PATCH] Changes made: *git-ignore file has been updated to avoid all .geojson files *details of new parameters have been added to the configtables (clean_osm_data_options.csv) *the config.default and config.tutorial files have been renamed and comments have been formated to fit PEP 8 recommendations *clean_osm_data script has been renamed and now the description of the added function follows PEP8 format and matches other functions *the proposed function in clean_osm_data has now been tested for substations also and is now being used in substations and cables --- .gitignore | 2 +- config.default.yaml | 12 ++--- config.tutorial.yaml | 12 ++--- doc/configtables/clean_osm_data_options.csv | 3 ++ scripts/clean_osm_data.py | 49 +++++++++------------ 5 files changed, 36 insertions(+), 42 deletions(-) diff --git a/.gitignore b/.gitignore index d359f263d..3e2a7d45e 100644 --- a/.gitignore +++ b/.gitignore @@ -23,7 +23,7 @@ dag.svg # Files appear for tests configs/scenarios/config.custom.yaml data/*.csv -data/custom_lines.geojson +data/*.geojson # Untracked folders (and files within) img/ diff --git a/config.default.yaml b/config.default.yaml index 168c539ec..65fcb035a 100644 --- a/config.default.yaml +++ b/config.default.yaml @@ -101,12 +101,12 @@ clean_osm_data_options: # osm = OpenStreetMap tag_substation: "transmission" # Filters only substations with 'transmission' tag, ('distribution' also available) add_line_endings: true # When "True", then line endings are added to the dataset of the substations generator_name_method: OSM # Methodology to specify the name to the generator. Options: OSM (name as by OSM dataset), closest_city (name by the closest city) - use_custom_lines: "OSM_only" # Flag added to use customized (Custom_only), OSM-downloaded (OSM_only), or both (Add_custom) in the clean_osm_data script (custom information should follow the format of the respective OSM .geojson file in the data folder) - custom_path_lines: False # Provide the specific absolute path of the custom file including its name and type e.g. (C:\...\...\pypsa-earth\data\custom_lines.geojson) - use_custom_substations: "OSM_only" # Flag added to use customized (Custom_only), OSM-downloaded (OSM_only), or both (Add_custom) in the clean_osm_data script (custom information should follow the format of the respective OSM .geojson file in the data folder) - custom_path_cables: False # Provide the specific absolute path of the custom file including its name and type e.g. (C:\...\...\pypsa-earth\data\custom_substations.geojson) - use_custom_substations: "OSM_only" # Flag added to use customized (Custom_only), OSM-downloaded (OSM_only), or both (Add_custom) in the clean_osm_data script (custom information should follow the format of the respective OSM .geojson file in the data folder) - custom_path_cables: False # Provide the specific absolute path of the custom file including its name and type e.g. (C:\...\...\pypsa-earth\data\custom_cables.geojson) + use_custom_lines: "OSM_only" # Use OSM (OSM_only), customized (Custom_only), or both data sets (Add_custom) + path_custom_lines: False # If exists, provide the specific absolute path of the custom file e.g. (C:\...\...\pypsa-earth\data\custom_lines.geojson) + use_custom_substations: "OSM_only" # Use OSM (OSM_only), customized (Custom_only), or both data sets (Add_custom) + path_custom_substations: False # If exists, provide the specific absolute path of the custom file e.g. (C:\...\...\pypsa-earth\data\custom_substations.geojson) + use_custom_cables: "OSM_only" # Use OSM (OSM_only), customized (Custom_only), or both data sets (Add_custom) + path_custom_cables: False # If exists, provide the specific absolute path of the custom file e.g. (C:\...\...\pypsa-earth\data\custom_cables.geojson) build_osm_network: # Options of the build_osm_network script; osm = OpenStreetMap group_close_buses: true # When "True", close buses are merged and guarantee the voltage matching among line endings diff --git a/config.tutorial.yaml b/config.tutorial.yaml index f1fbf1aa7..27c40a186 100644 --- a/config.tutorial.yaml +++ b/config.tutorial.yaml @@ -115,12 +115,12 @@ clean_osm_data_options: tag_substation: "transmission" # needed feature tag to be considered for the analysis. If empty, no filtering on the tag_substation is performed add_line_endings: true # When true, the line endings are added to the dataset of the substations generator_name_method: OSM # Methodology to specify the name to the generator. Options: OSM (name as by OSM dataset), closest_city (name by the closest city) - use_custom_lines: "OSM_only" # Flag added to use customized (Custom_only), OSM-downloaded (OSM_only), or both (Add_custom) in the clean_osm_data script (custom information should follow the format of the respective OSM .geojson file in the data folder) - custom_path_lines: false # Provide the specific absolute path of the custom file including its name and type e.g. (C:\...\...\pypsa-earth\data\custom_lines.geojson) - use_custom_substations: "OSM_only" # Flag added to use customized (Custom_only), OSM-downloaded (OSM_only), or both (Add_custom) in the clean_osm_data script (custom information should follow the format of the respective OSM .geojson file in the data folder) - custom_path_substations: false # Provide the specific absolute path of the custom file including its name and type e.g. (C:\...\...\pypsa-earth\data\custom_substations.geojson) - use_custom_cables: "OSM_only" # Flag added to use customized (Custom_only), OSM-downloaded (OSM_only), or both (Add_custom) in the clean_osm_data script (custom information should follow the format of the respective OSM .geojson file in the data folder) - custom_path_cables: false # Provide the specific absolute path of the custom file including its name and type e.g. (C:\...\...\pypsa-earth\data\custom_cables.geojson) + use_custom_lines: "OSM_only" # Use OSM (OSM_only), customized (Custom_only), or both data sets (Add_custom) + path_custom_lines: False # If exists, provide the specific absolute path of the custom file e.g. (C:\...\...\pypsa-earth\data\custom_lines.geojson) + use_custom_substations: "OSM_only" # Use OSM (OSM_only), customized (Custom_only), or both data sets (Add_custom) + path_custom_substations: False # If exists, provide the specific absolute path of the custom file e.g. (C:\...\...\pypsa-earth\data\custom_substations.geojson) + use_custom_cables: "OSM_only" # Use OSM (OSM_only), customized (Custom_only), or both data sets (Add_custom) + path_custom_cables: False # If exists, provide the specific absolute path of the custom file e.g. (C:\...\...\pypsa-earth\data\custom_cables.geojson) build_osm_network: # Options of the build_osm_network script; osm = OpenStreetMap group_close_buses: true # When "True", close buses are merged and guarantee the voltage matching among line endings diff --git a/doc/configtables/clean_osm_data_options.csv b/doc/configtables/clean_osm_data_options.csv index 5e499dcf9..c737ae3e1 100644 --- a/doc/configtables/clean_osm_data_options.csv +++ b/doc/configtables/clean_osm_data_options.csv @@ -4,3 +4,6 @@ threshold_voltage, V,, "Assets below the voltage threshold will not be considere tag_substation,, "{""transmission"", ""distribution""}", "Filters only substations with the corresponding tag (""transmission"" or ""distribution"")." add_line_endings, bool, "{True, False}", "True: line endings are added to the dataset of the substations." generator_name_method,, "{OSM, closest_city}", "Methodology to specify the name of generators. From OpenStreetMap (OSM) or named after the closest city (closest_city)." +use_custom_lines,,"{OSM_only, Custom_only, Add_custom}", "Method for selecting which type of data is used for lines in the clean_osm_data script. Use OSM (OSM_only), customized (Custom_only), or both data sets (Add_custom)." +use_custom_cables,,"{OSM_only, Custom_only, Add_custom}", "Method for selecting which type of data is used for cables in the clean_osm_data script. Use OSM (OSM_only), customized (Custom_only), or both data sets (Add_custom)." +use_custom_substations,,"{OSM_only, Custom_only, Add_custom}", "Method for selecting which type of data is used for substations in the clean_osm_data script. Use OSM (OSM_only), customized (Custom_only), or both data sets (Add_custom)." \ No newline at end of file diff --git a/scripts/clean_osm_data.py b/scripts/clean_osm_data.py index fe522141b..1a558e84a 100644 --- a/scripts/clean_osm_data.py +++ b/scripts/clean_osm_data.py @@ -835,33 +835,24 @@ def set_name_by_closestcity(df_all_generators, colname="name"): return df_all_generators -def use_custom_data_files( - custom_component_type, -): #### custom_data_type should be a string named lines, cables or substations - # checks the type of data/components to be reviewed - if custom_component_type == "lines": - custom_conditional = snakemake.config["clean_osm_data_options"][ - "use_custom_lines" - ] - custom_path = snakemake.config["clean_osm_data_options"].get( - "custom_path_lines", False - ) - - elif custom_component_type == "substations": - custom_conditional = snakemake.config["clean_osm_data_options"][ - "use_custom_substations" - ] - custom_path = snakemake.config["clean_osm_data_options"].get( - "custom_path_substations", False - ) - - elif custom_component_type == "cables": - custom_conditional = snakemake.config["clean_osm_data_options"][ - "use_custom_cables" - ] - custom_path = snakemake.config["clean_osm_data_options"].get( - "custom_path_cables", False - ) +def use_custom_data_files (custom_component_type): + """ + Function to check if OSM or custom data should be considered. + the custom_component_type should be a string named lines, cables or substations + """ + + #checks the type of data/components to be reviewed + if custom_component_type == 'lines' : + custom_conditional = snakemake.config["clean_osm_data_options"]["use_custom_lines"] + custom_path = snakemake.config["clean_osm_data_options"].get("path_custom_lines",False) + + elif custom_component_type == 'substations' : + custom_conditional = snakemake.config["clean_osm_data_options"]["use_custom_substations"] + custom_path = snakemake.config["clean_osm_data_options"].get("path_custom_substations",False) + + elif custom_component_type == 'cables' : + custom_conditional = snakemake.config["clean_osm_data_options"]["use_custom_cables"] + custom_path = snakemake.config["clean_osm_data_options"].get("path_custom_cables",False) else: raise ValueError( @@ -922,7 +913,7 @@ def clean_data( if os.path.getsize(input_files["cables"]) > 0: logger.info("Add OSM cables to data") # Load raw data lines - df_cables = gpd.read_file(input_files["cables"]) + df_cables = use_custom_data_files ('cables') # prepare cables dataframe and data types df_cables = prepare_lines_df(df_cables) @@ -970,7 +961,7 @@ def clean_data( logger.info("Process OSM substations") if os.path.getsize(input_files["substations"]) > 0: - df_all_substations = gpd.read_file(input_files["substations"]) + df_all_substations = use_custom_data_files ('substations') # prepare dataset for substations df_all_substations = prepare_substation_df(df_all_substations)