From d193745d5d68d2d285204a1c5fa5dd40918de93a Mon Sep 17 00:00:00 2001
From: Emre-Yorat89 <yoratemre@gmail.com>
Date: Fri, 20 Sep 2024 16:30:10 +0300
Subject: [PATCH 1/7] building data download with an overpass query v2

---
 Snakefile                    |  2 +-
 config.distribution.yaml     |  1 +
 scripts/download_osm_data.py | 96 ++++++++++++++++++++++++++----------
 3 files changed, 72 insertions(+), 27 deletions(-)

diff --git a/Snakefile b/Snakefile
index 9d4cddc..89bd58e 100644
--- a/Snakefile
+++ b/Snakefile
@@ -159,7 +159,7 @@ rule create_network:
         "scripts/create_network.py"
 
 
-if config["enable"].get("download_osm_buildings", True):
+if config["enable"].get("download_osm_buildings", True) or config["enable"].get("download_osm_buildings_overpass", True):
 
     rule download_osm_data:
         output:
diff --git a/config.distribution.yaml b/config.distribution.yaml
index a721f24..7d05b20 100644
--- a/config.distribution.yaml
+++ b/config.distribution.yaml
@@ -18,6 +18,7 @@ enable:
   retrieve_cost_data: true
   download_osm_data: true
   download_osm_buildings: false
+  download_osm_buildings_overpass: false
   # If "build_cutout" : true # requires cds API key
   # https://cds.climate.copernicus.eu/api-how-to
   # More information
diff --git a/scripts/download_osm_data.py b/scripts/download_osm_data.py
index 4917662..afaa0b8 100644
--- a/scripts/download_osm_data.py
+++ b/scripts/download_osm_data.py
@@ -3,6 +3,8 @@
 import os
 import shutil
 from pathlib import Path
+import json
+import requests
 
 import yaml
 from _helpers_dist import configure_logging, create_logger, read_osm_config
@@ -64,6 +66,38 @@ def convert_iso_to_geofk(
     else:
         return iso_code
 
+def retrieve_osm_data_overpass(coordinates, features, url, path):
+    """
+    The buildings inside the specified coordinates are retrieved by using overpass API. 
+    The region coordinates should be defined in the config.yaml file.
+    Parameters
+    ----------
+    coordinates : dict
+        Coordinates of the rectangular region where buildings to be downloaded from osm resides. 
+    features : str
+        The feature that is searched in the osm database
+    url : str
+        osm query address
+    path : str
+        the directory where the buildings are going to be downloaded.
+    """
+    
+    out_format = "json"
+    for item in coordinates.keys():
+        
+        overpass_query = f'''
+        [out:json];
+        node[{features}]({coordinates[item]["lon_min"]}, {coordinates[item]["lat_min"]}, {coordinates[item]["lon_max"]}, {coordinates[item]["lat_max"]});
+        out;
+        '''
+        try:
+            response = requests.get(url, params={'data': overpass_query})
+            response.raise_for_status()
+            outpath = Path.joinpath(path, f"all_raw_building_{item}.{out_format}")
+            with open(outpath, "w") as out_file:
+                json.dump(response.json(), out_file, indent = 2)
+        except (json.JSONDecodeError, requests.exceptions.RequestException) as e:
+            logger.error(f"Error downloading osm data for the specified coordinates")
 
 if __name__ == "__main__":
     if "snakemake" not in globals():
@@ -81,29 +115,39 @@ def convert_iso_to_geofk(
     countries = snakemake.config["countries"]
     country_list = country_list_to_geofk(countries)
 
-    eo.save_osm_data(
-        region_list=country_list,
-        primary_name="building",
-        feature_list=["ALL"],
-        update=False,
-        mp=False,
-        data_dir=store_path_data,
-        out_dir=store_path_resources,
-        out_format=["csv", "geojson"],
-        out_aggregate=True,
-    )
-
-    out_path = Path.joinpath(store_path_resources, "out")
-    out_formats = ["csv", "geojson"]
-    new_files = os.listdir(out_path)
-
-    for f in out_formats:
-        new_file_name = Path.joinpath(store_path_resources, f"all_raw_building.{f}")
-        old_file = list(Path(out_path).glob(f"*building.{f}"))
-
-        if not old_file:
-            with open(new_file_name, "w") as f:
-                pass
-        else:
-            logger.info(f"Move {old_file[0]} to {new_file_name}")
-            shutil.move(old_file[0], new_file_name)
+    if snakemake.config["enable"]["download_osm_buildings"] == True:
+        eo.save_osm_data(
+            region_list=country_list,
+            primary_name="building",
+            feature_list=["ALL"],
+            update=False,
+            mp=False,
+            data_dir=store_path_data,
+            out_dir=store_path_resources,
+            out_format=["csv", "geojson"],
+            out_aggregate=True,
+        )
+
+        out_path = Path.joinpath(store_path_resources, "out")
+        out_formats = ["csv", "geojson"]
+        new_files = os.listdir(out_path)
+
+        for f in out_formats:
+            new_file_name = Path.joinpath(store_path_resources, f"all_raw_building.{f}")
+            old_file = list(Path(out_path).glob(f"*building.{f}"))
+
+            if not old_file:
+                with open(new_file_name, "w") as f:
+                    pass
+            else:
+                logger.info(f"Move {old_file[0]} to {new_file_name}")
+                shutil.move(old_file[0], new_file_name)
+
+    if snakemake.config["enable"]["download_osm_buildings_overpass"] == True:
+        microgrids_list = snakemake.config["microgrids_list"]
+        features = "building"
+        overpass_url = 'https://overpass-api.de/api/interpreter'
+        retrieve_osm_data_overpass(microgrids_list, features, overpass_url, store_path_resources)
+        outpath = Path.joinpath(store_path_resources, "all_raw_building.geojson")
+        with open(outpath, 'w') as fp: # an empty .geojson file is created to bypass snakemake output file requirement in the download_osm rule.
+            pass
\ No newline at end of file

From 8b67af6a49f21bb7c86274538c52da2b0708eaad Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Fri, 20 Sep 2024 13:43:24 +0000
Subject: [PATCH 2/7] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 Snakefile                    |  4 +++-
 scripts/download_osm_data.py | 34 ++++++++++++++++++++--------------
 2 files changed, 23 insertions(+), 15 deletions(-)

diff --git a/Snakefile b/Snakefile
index 89bd58e..9bc8681 100644
--- a/Snakefile
+++ b/Snakefile
@@ -159,7 +159,9 @@ rule create_network:
         "scripts/create_network.py"
 
 
-if config["enable"].get("download_osm_buildings", True) or config["enable"].get("download_osm_buildings_overpass", True):
+if config["enable"].get("download_osm_buildings", True) or config["enable"].get(
+    "download_osm_buildings_overpass", True
+):
 
     rule download_osm_data:
         output:
diff --git a/scripts/download_osm_data.py b/scripts/download_osm_data.py
index afaa0b8..e1ab16e 100644
--- a/scripts/download_osm_data.py
+++ b/scripts/download_osm_data.py
@@ -1,11 +1,11 @@
 # -*- coding: utf-8 -*-
+import json
 import logging
 import os
 import shutil
 from pathlib import Path
-import json
-import requests
 
+import requests
 import yaml
 from _helpers_dist import configure_logging, create_logger, read_osm_config
 from earth_osm import eo
@@ -66,14 +66,15 @@ def convert_iso_to_geofk(
     else:
         return iso_code
 
+
 def retrieve_osm_data_overpass(coordinates, features, url, path):
     """
-    The buildings inside the specified coordinates are retrieved by using overpass API. 
+    The buildings inside the specified coordinates are retrieved by using overpass API.
     The region coordinates should be defined in the config.yaml file.
     Parameters
     ----------
     coordinates : dict
-        Coordinates of the rectangular region where buildings to be downloaded from osm resides. 
+        Coordinates of the rectangular region where buildings to be downloaded from osm resides.
     features : str
         The feature that is searched in the osm database
     url : str
@@ -81,24 +82,25 @@ def retrieve_osm_data_overpass(coordinates, features, url, path):
     path : str
         the directory where the buildings are going to be downloaded.
     """
-    
+
     out_format = "json"
     for item in coordinates.keys():
-        
-        overpass_query = f'''
+
+        overpass_query = f"""
         [out:json];
         node[{features}]({coordinates[item]["lon_min"]}, {coordinates[item]["lat_min"]}, {coordinates[item]["lon_max"]}, {coordinates[item]["lat_max"]});
         out;
-        '''
+        """
         try:
-            response = requests.get(url, params={'data': overpass_query})
+            response = requests.get(url, params={"data": overpass_query})
             response.raise_for_status()
             outpath = Path.joinpath(path, f"all_raw_building_{item}.{out_format}")
             with open(outpath, "w") as out_file:
-                json.dump(response.json(), out_file, indent = 2)
+                json.dump(response.json(), out_file, indent=2)
         except (json.JSONDecodeError, requests.exceptions.RequestException) as e:
             logger.error(f"Error downloading osm data for the specified coordinates")
 
+
 if __name__ == "__main__":
     if "snakemake" not in globals():
         from _helpers_dist import mock_snakemake, sets_path_to_root
@@ -146,8 +148,12 @@ def retrieve_osm_data_overpass(coordinates, features, url, path):
     if snakemake.config["enable"]["download_osm_buildings_overpass"] == True:
         microgrids_list = snakemake.config["microgrids_list"]
         features = "building"
-        overpass_url = 'https://overpass-api.de/api/interpreter'
-        retrieve_osm_data_overpass(microgrids_list, features, overpass_url, store_path_resources)
+        overpass_url = "https://overpass-api.de/api/interpreter"
+        retrieve_osm_data_overpass(
+            microgrids_list, features, overpass_url, store_path_resources
+        )
         outpath = Path.joinpath(store_path_resources, "all_raw_building.geojson")
-        with open(outpath, 'w') as fp: # an empty .geojson file is created to bypass snakemake output file requirement in the download_osm rule.
-            pass
\ No newline at end of file
+        with open(
+            outpath, "w"
+        ) as fp:  # an empty .geojson file is created to bypass snakemake output file requirement in the download_osm rule.
+            pass

From 1ebed853d054a5f7b34df950d369e3d00b591ee0 Mon Sep 17 00:00:00 2001
From: Emre-Yorat89 <yoratemre@gmail.com>
Date: Sun, 20 Oct 2024 15:29:02 +0300
Subject: [PATCH 3/7] building data download with an overpass query v3

---
 Snakefile                    | 4 +---
 config.distribution.yaml     | 2 +-
 scripts/download_osm_data.py | 4 ++--
 3 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/Snakefile b/Snakefile
index 9bc8681..9d4cddc 100644
--- a/Snakefile
+++ b/Snakefile
@@ -159,9 +159,7 @@ rule create_network:
         "scripts/create_network.py"
 
 
-if config["enable"].get("download_osm_buildings", True) or config["enable"].get(
-    "download_osm_buildings_overpass", True
-):
+if config["enable"].get("download_osm_buildings", True):
 
     rule download_osm_data:
         output:
diff --git a/config.distribution.yaml b/config.distribution.yaml
index 7d05b20..08fd46b 100644
--- a/config.distribution.yaml
+++ b/config.distribution.yaml
@@ -18,7 +18,7 @@ enable:
   retrieve_cost_data: true
   download_osm_data: true
   download_osm_buildings: false
-  download_osm_buildings_overpass: false
+  download_osm_method: overpass # or earth_osm
   # If "build_cutout" : true # requires cds API key
   # https://cds.climate.copernicus.eu/api-how-to
   # More information
diff --git a/scripts/download_osm_data.py b/scripts/download_osm_data.py
index e1ab16e..125766b 100644
--- a/scripts/download_osm_data.py
+++ b/scripts/download_osm_data.py
@@ -117,7 +117,7 @@ def retrieve_osm_data_overpass(coordinates, features, url, path):
     countries = snakemake.config["countries"]
     country_list = country_list_to_geofk(countries)
 
-    if snakemake.config["enable"]["download_osm_buildings"] == True:
+    if snakemake.config["enable"]["download_osm_method"] == "earth_osm":
         eo.save_osm_data(
             region_list=country_list,
             primary_name="building",
@@ -145,7 +145,7 @@ def retrieve_osm_data_overpass(coordinates, features, url, path):
                 logger.info(f"Move {old_file[0]} to {new_file_name}")
                 shutil.move(old_file[0], new_file_name)
 
-    if snakemake.config["enable"]["download_osm_buildings_overpass"] == True:
+    elif snakemake.config["enable"]["download_osm_method"] == "overpass":
         microgrids_list = snakemake.config["microgrids_list"]
         features = "building"
         overpass_url = "https://overpass-api.de/api/interpreter"

From ddac58640f7260ac52aec71615230a376d2ea310 Mon Sep 17 00:00:00 2001
From: Margherita Capitani <m.capitani2@studenti.unipi.it>
Date: Wed, 27 Nov 2024 13:12:05 +0100
Subject: [PATCH 4/7] Fix_to_finalize_Emre_PR

---
 config.distribution.yaml     |  2 +-
 scripts/cluster_buildings.py | 14 ++++-----
 scripts/download_osm_data.py | 61 ++++++++++++++++++++++++++----------
 3 files changed, 53 insertions(+), 24 deletions(-)

diff --git a/config.distribution.yaml b/config.distribution.yaml
index 08fd46b..6111a72 100644
--- a/config.distribution.yaml
+++ b/config.distribution.yaml
@@ -17,7 +17,7 @@ enable:
   retrieve_databundle: true
   retrieve_cost_data: true
   download_osm_data: true
-  download_osm_buildings: false
+  download_osm_buildings: true
   download_osm_method: overpass # or earth_osm
   # If "build_cutout" : true # requires cds API key
   # https://cds.climate.copernicus.eu/api-how-to
diff --git a/scripts/cluster_buildings.py b/scripts/cluster_buildings.py
index 3c2c254..f94d80f 100644
--- a/scripts/cluster_buildings.py
+++ b/scripts/cluster_buildings.py
@@ -19,14 +19,14 @@
 _logger.setLevel(logging.INFO)
 
 
-def buildings_classification(input_file, crs, house_area_limit):
+def buildings_classification(input_file, crs):
     """
     Filters the data contained in all_raw_building, selecting only Polygon elements,
     after which the plan area is calculated for each building with the specified coordinate system
     and adds the information to the geodataframe.
     """
     microgrid_buildings = gpd.read_file(input_file)
-    microgrid_buildings.rename(columns={"tags.building": "tags_building"}, inplace=True)
+    microgrid_buildings.rename(columns={"building": "tags_building"}, inplace=True)
     microgrid_buildings = microgrid_buildings.loc[
         microgrid_buildings.geometry.type != "Point"
     ]
@@ -54,9 +54,7 @@ def get_central_points_geojson_with_buildings(
     a dataframe with all of the buildings divided into clusters,
     a csv file where for each cluster the building types are counted
     """
-    microgrid_buildings = buildings_classification(
-        input_filepath, crs, house_area_limit
-    )
+    microgrid_buildings = buildings_classification(input_filepath, crs)
     centroids_building = [
         (row.geometry.centroid.x, row.geometry.centroid.y)
         for row in microgrid_buildings.itertuples()
@@ -79,8 +77,10 @@ def get_central_points_geojson_with_buildings(
                 "cluster": i,
             }
         )
-    central_features = gpd.GeoDataFrame(central_features, crs=microgrid_buildings.crs)
-    central_features.to_file(output_filepath_centroids)
+    central_features = gpd.GeoDataFrame(
+        central_features, crs=microgrid_buildings.crs
+    ).to_crs("EPSG:4326")
+    central_features.to_file(output_filepath_centroids, driver="GeoJSON")
 
     clusters = []
     for i, row in enumerate(microgrid_buildings.itertuples()):
diff --git a/scripts/download_osm_data.py b/scripts/download_osm_data.py
index 125766b..d6697a3 100644
--- a/scripts/download_osm_data.py
+++ b/scripts/download_osm_data.py
@@ -67,7 +67,7 @@ def convert_iso_to_geofk(
         return iso_code
 
 
-def retrieve_osm_data_overpass(coordinates, features, url, path):
+def retrieve_osm_data_geojson(coordinates, features, url, path):
     """
     The buildings inside the specified coordinates are retrieved by using overpass API.
     The region coordinates should be defined in the config.yaml file.
@@ -80,23 +80,58 @@ def retrieve_osm_data_overpass(coordinates, features, url, path):
     url : str
         osm query address
     path : str
-        the directory where the buildings are going to be downloaded.
+        Directory where the GeoJSON file will be saved.
     """
 
-    out_format = "json"
     for item in coordinates.keys():
 
         overpass_query = f"""
         [out:json];
-        node[{features}]({coordinates[item]["lon_min"]}, {coordinates[item]["lat_min"]}, {coordinates[item]["lon_max"]}, {coordinates[item]["lat_max"]});
-        out;
+        way["{features}"]({coordinates[item]["lat_min"]}, {coordinates[item]["lon_min"]}, {coordinates[item]["lat_max"]}, {coordinates[item]["lon_max"]});
+        (._;>;);
+        out body;
         """
+
         try:
+            # Send request to API Overpass
             response = requests.get(url, params={"data": overpass_query})
             response.raise_for_status()
-            outpath = Path.joinpath(path, f"all_raw_building_{item}.{out_format}")
-            with open(outpath, "w") as out_file:
-                json.dump(response.json(), out_file, indent=2)
+            data = response.json()
+            # Create a dictionary to map nodes with their coordinates
+            node_coordinates = {
+                node["id"]: [node["lon"], node["lat"]]
+                for node in data["elements"]
+                if node["type"] == "node"
+            }
+            # Choose the output path to save the file.
+            outpath = Path(path) / f"all_raw_building.geojson"
+            # outpath = Path(path) / f"all_raw_building_{item}.geojson" #ATTENTION: Currently the other parts of the code ( clean earth osm data,cluster building, and others) have not been updated to run on multiple microgrids simultaneously. For now we do not exploit this to run the code. As soon as we update the other parts of the code as well, we will exploit it.
+            outpath.parent.mkdir(parents=True, exist_ok=True)
+            # Write the geojson file
+            with open(outpath, "w") as f:
+                f.write('{"type":"FeatureCollection","features":[\n')
+                features = []
+                for element in data["elements"]:
+                    if element["type"] == "way" and "nodes" in element:
+                        coordinates = [
+                            node_coordinates[node_id]
+                            for node_id in element["nodes"]
+                            if node_id in node_coordinates
+                        ]
+                        properties = {"id": element["id"]}
+                        if "tags" in element:
+                            properties.update(element["tags"])
+                        feature = {
+                            "type": "Feature",
+                            "properties": properties,
+                            "geometry": {
+                                "type": "Polygon",
+                                "coordinates": [coordinates],
+                            },
+                        }
+                        features.append(json.dumps(feature, separators=(",", ":")))
+                f.write(",\n".join(features))
+                f.write("\n]}\n")
         except (json.JSONDecodeError, requests.exceptions.RequestException) as e:
             logger.error(f"Error downloading osm data for the specified coordinates")
 
@@ -149,11 +184,5 @@ def retrieve_osm_data_overpass(coordinates, features, url, path):
         microgrids_list = snakemake.config["microgrids_list"]
         features = "building"
         overpass_url = "https://overpass-api.de/api/interpreter"
-        retrieve_osm_data_overpass(
-            microgrids_list, features, overpass_url, store_path_resources
-        )
-        outpath = Path.joinpath(store_path_resources, "all_raw_building.geojson")
-        with open(
-            outpath, "w"
-        ) as fp:  # an empty .geojson file is created to bypass snakemake output file requirement in the download_osm rule.
-            pass
+        output_file = Path.cwd() / "resources" / RDIR / "osm" / "raw"
+        retrieve_osm_data_geojson(microgrids_list, features, overpass_url, output_file)

From a9f857ef5cfcd2b9cf9f1a7ef464a0428607a9d6 Mon Sep 17 00:00:00 2001
From: Margherita Capitani <m.capitani2@studenti.unipi.it>
Date: Wed, 27 Nov 2024 13:26:31 +0100
Subject: [PATCH 5/7] Fix_test_config

---
 test/config.distribution.test.yaml | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/test/config.distribution.test.yaml b/test/config.distribution.test.yaml
index d3aa18c..db0b3f4 100644
--- a/test/config.distribution.test.yaml
+++ b/test/config.distribution.test.yaml
@@ -25,9 +25,12 @@ enable:
   retrieve_databundle: true
   retrieve_cost_data: true
   download_osm_data: true
-  download_osm_buildings: false
-  # If "build_cutout" : true # requires cds API key https://cds.climate.copernicus.eu/api-how-to
-  # More information https://atlite.readthedocs.io/en/latest/introduction.html#datasets
+  download_osm_buildings: true
+  download_osm_method: overpass # or earth_osm
+  # If "build_cutout" : true # requires cds API key
+  # https://cds.climate.copernicus.eu/api-how-to
+  # More information
+  # https://atlite.readthedocs.io/en/latest/introduction.html#datasets
   build_cutout: false
   build_natura_raster: false  # If True, then build_natura_raster can be run
 

From ebfce84746cc0d12fa1ab2b6894a1cb41daf1c6c Mon Sep 17 00:00:00 2001
From: Margherita Capitani
 <165152575+Margherita-Capitani@users.noreply.github.com>
Date: Wed, 4 Dec 2024 17:47:11 +0100
Subject: [PATCH 6/7] Update release_notes.rst

---
 doc/release_notes.rst | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/doc/release_notes.rst b/doc/release_notes.rst
index de24c5f..cab35a6 100644
--- a/doc/release_notes.rst
+++ b/doc/release_notes.rst
@@ -10,7 +10,11 @@ Release Notes
 
 Upcoming Release
 ================
+New available options
 
+	 * The energy demand time series can now be determined using a new methodology based on social inputs that integrates with the RAMP tool.
+
+	*	Automated downloading of buildings within the microgrid is now supported through the new download_osm_data rule.
 
 
 

From f3df960b46de305df508c1e0087768b81b583838 Mon Sep 17 00:00:00 2001
From: Davide Fioriti <67809479+davide-f@users.noreply.github.com>
Date: Wed, 11 Dec 2024 14:32:16 +0100
Subject: [PATCH 7/7] Update release_notes.rst

---
 doc/release_notes.rst | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/doc/release_notes.rst b/doc/release_notes.rst
index cab35a6..eff1c3c 100644
--- a/doc/release_notes.rst
+++ b/doc/release_notes.rst
@@ -10,11 +10,10 @@ Release Notes
 
 Upcoming Release
 ================
-New available options
 
-	 * The energy demand time series can now be determined using a new methodology based on social inputs that integrates with the RAMP tool.
+* The energy demand time series can now be determined using a new methodology based on social inputs that integrates with the RAMP tool. `PR #55 <https://github.com/pypsa-meets-earth/pypsa-distribution/pull/55>`__
 
-	*	Automated downloading of buildings within the microgrid is now supported through the new download_osm_data rule.
+* Automated downloading of buildings within the microgrid is now supported through the new download_osm_data rule. `PR #52 <https://github.com/pypsa-meets-earth/pypsa-distribution/pull/52>`__ and `PR #56 <https://github.com/pypsa-meets-earth/pypsa-distribution/pull/56>`__