Merge branch 'main' of https://github.com/pypsa-meets-earth/pypsa-dis…

…tribution

.pre-commit-config.yaml

@@ -9,7 +9,7 @@ exclude: ^(LICENSES|README.md)
 
 repos:
 - repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: v4.4.0
+  rev: v4.5.0
   hooks:
   - id: check-merge-conflict
   - id: end-of-file-fixer
@@ -36,7 +36,7 @@ repos:
 
 # Formatting with "black" coding style
 - repo: https://github.com/psf/black
-  rev: 23.7.0
+  rev: 23.10.0
   hooks:
   # Format Python files
   - id: black
@@ -45,7 +45,7 @@ repos:
 
 # Do YAML formatting (before the linter checks it for misses)
 - repo: https://github.com/macisamuele/language-formatters-pre-commit-hooks
-  rev: v2.10.0
+  rev: v2.11.0
   hooks:
   - id: pretty-format-yaml
     args: [--autofix, --indent, '2', --preserve-quotes]
@@ -59,7 +59,7 @@ repos:
 
 # Format Snakemake rule / workflow files
 - repo: https://github.com/snakemake/snakefmt
-  rev: v0.8.4
+  rev: v0.8.5
   hooks:
   - id: snakefmt
 

Snakefile

@@ -53,6 +53,11 @@ subworkflow pypsaearth:
         "./config.yaml"
 
 
+rule clean:
+    run:
+        shell("snakemake -j 1 solve_network --delete-all-output")
+
+
 rule build_demand:
     input:
         sample_profile=PROFILE,

scripts/build_demand.py

@@ -35,12 +35,14 @@
 import json
 import logging
 import os
+import shutil
 
 import geopandas as gpd
 import pandas as pd
 import pypsa
 import rasterio
 import rasterio.mask
+import requests
 from _helpers_dist import (
     configure_logging,
     sets_path_to_root,
@@ -51,10 +53,12 @@
 _logger.setLevel(logging.INFO)
 
 
-def get_WorldPop_path(
+def get_WorldPop_data(
     country_code,
     year,
-    out_logging,
+    update=False,
+    out_logging=False,
+    size_min=300,
 ):
     """
     Download tiff file for each country code using the standard method from worldpop datastore with 1kmx1km resolution.
@@ -66,26 +70,66 @@ def get_WorldPop_path(
         Files downloaded from https://data.worldpop.org/ datasets WorldPop UN adjusted
     year : int
         Year of the data to download
-
+    update : bool
+        Update = true, forces re-download of files
+    size_min : int
+        Minimum size of each file to download
     Returns
     -------
     WorldPop_inputfile : str
         Path of the file
     """
 
-    if out_logging:
-        _logger.info("Download WorldPop datasets")
-
     three_digits_code = two_2_three_digits_country(country_code)
 
-    return os.path.join(
+    if out_logging:
+        _logger.info("Get WorldPop datasets")
+
+    if country_code == "XK":
+        WorldPop_filename = f"srb_ppp_{year}_UNadj_constrained.tif"
+        WorldPop_urls = [
+            f"https://data.worldpop.org/GIS/Population/Global_2000_2020_Constrained/2020/BSGM/SRB/{WorldPop_filename}",
+            f"https://data.worldpop.org/GIS/Population/Global_2000_2020_Constrained/2020/maxar_v1/SRB/{WorldPop_filename}",
+        ]
+    else:
+        WorldPop_filename = (
+            f"{three_digits_code.lower()}_ppp_{year}_UNadj_constrained.tif"
+        )
+        # Urls used to possibly download the file
+        WorldPop_urls = [
+            f"https://data.worldpop.org/GIS/Population/Global_2000_2020_Constrained/2020/BSGM/{two_2_three_digits_country(country_code).upper()}/{WorldPop_filename}",
+            f"https://data.worldpop.org/GIS/Population/Global_2000_2020_Constrained/2020/maxar_v1/{two_2_three_digits_country(country_code).upper()}/{WorldPop_filename}",
+        ]
+
+    WorldPop_inputfile = os.path.join(
         os.getcwd(),
         "pypsa-earth",
         "data",
         "WorldPop",
-        f"{three_digits_code.lower()}_ppp_{year}_UNadj_constrained.tif",
+        WorldPop_filename,
     )  # Input filepath tif
 
+    if not os.path.exists(WorldPop_inputfile) or update is True:
+        if out_logging:
+            _logger.warning(
+                f"{WorldPop_filename} does not exist, downloading to {WorldPop_inputfile}"
+            )
+        #  create data/osm directory
+        os.makedirs(os.path.dirname(WorldPop_inputfile), exist_ok=True)
+
+        loaded = False
+        for WorldPop_url in WorldPop_urls:
+            with requests.get(WorldPop_url, stream=True) as r:
+                with open(WorldPop_inputfile, "wb") as f:
+                    if float(r.headers["Content-length"]) > size_min:
+                        shutil.copyfileobj(r.raw, f)
+                        loaded = True
+                        break
+        if not loaded:
+            _logger.error(f"Impossible to download {WorldPop_filename}")
+
+    return WorldPop_inputfile, WorldPop_filename
+
 
 # Estimate the total population of tghe microgrid
 def estimate_microgrid_population(
@@ -208,7 +252,7 @@ def calculate_load(
         len(snakemake.config["countries"]) == 1
     ), "Error: only a country shall be specified"
 
-    worldpop_path = get_WorldPop_path(
+    worldpop_path, worldpop_flname = get_WorldPop_data(
         snakemake.config["countries"][
             0
         ],  # TODO: this needs fix to generalize the countries

scripts/build_shapes.py

@@ -79,8 +79,8 @@ def create_bus_regions(microgrids_list, output_path):
         microgrid_names.append(microgrid_name)
 
         # Centre of the rectangle of the microgrid
-        x = 7.499138
-        y = 9.106406
+        x = (values[0] + values[1]) / 2
+        y = (values[2] + values[3]) / 2
         microgrid_x.append(x)
         microgrid_y.append(y)