merge origin/master

.github/workflows/test.yaml

@@ -85,6 +85,11 @@ jobs:
         conda env update -n pypsa-eur -f ${{ env.env_file }}
         echo "Run conda list" && conda list
 
+    - name: Run pylint check on scripts
+      # check for undefined variables to reuse functions across scripts
+      run: |
+        pylint --disable=all --enable=E0601 --output-format=parseable scripts/add_* scripts/prepare_* scripts/solve_* 
+
     - name: Run snakemake test workflows
       run: |
         make test

.github/workflows/validate.yaml

@@ -14,19 +14,33 @@ jobs:
     name: Run validation
     if: github.event.pull_request.head.repo.full_name == github.repository
     runs-on: self-hosted
+    outputs:
+      validation_failed: ${{ steps.check_validation.outputs.failed }}
     steps:
     - uses: lkstrp/[email protected]
+      id: validation
+      continue-on-error: true
       with:
         step: run-self-hosted-validation
         env_file: envs/environment.yaml
         snakemake_config: config/test/config.validator.yaml
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    - name: Check validation status
+      id: check_validation
+      run: |
+        if [[ "${{ steps.validation.outcome }}" == "failure" ]]; then
+          echo "failed=true" >> $GITHUB_OUTPUT
+        else
+          echo "failed=false" >> $GITHUB_OUTPUT
+        fi
 
   create-report:
     name: Create report
-    if: github.event.pull_request.head.repo.full_name == github.repository
     needs: run-validation
+    if: |
+      github.event.pull_request.head.repo.full_name == github.repository && 
+      needs.run-validation.outputs.validation_failed != 'true'
     runs-on: ubuntu-latest
     steps:
     - uses: lkstrp/[email protected]

.pre-commit-config.yaml

@@ -14,7 +14,7 @@ repos:
 # Run ruff to lint and format
 - repo: https://github.com/astral-sh/ruff-pre-commit
   # Ruff version.
-  rev: v0.9.3
+  rev: v0.9.4
   hooks:
     # Run the linter.
   - id: ruff
@@ -24,7 +24,7 @@ repos:
 
   # Find common spelling mistakes in comments and docstrings
 - repo: https://github.com/codespell-project/codespell
-  rev: v2.4.0
+  rev: v2.4.1
   hooks:
   - id: codespell
     args: ['--ignore-regex="(\b[A-Z]+\b)"', '--ignore-words-list=fom,appartment,bage,ore,setis,tabacco,berfore,vor,pris'] # Ignore capital case words, e.g. country codes

Snakefile

@@ -4,13 +4,19 @@
 
 from pathlib import Path
 import yaml
-from os.path import normpath, exists
+from os.path import normpath, exists, join
 from shutil import copyfile, move, rmtree
 from snakemake.utils import min_version
 
 min_version("8.11")
 
-from scripts._helpers import path_provider, copy_default_files, get_scenarios, get_rdir
+from scripts._helpers import (
+    path_provider,
+    copy_default_files,
+    get_scenarios,
+    get_rdir,
+    get_shadow,
+)
 
 
 copy_default_files(workflow)
@@ -23,14 +29,16 @@ configfile: "config/config.yaml"
 run = config["run"]
 scenarios = get_scenarios(run)
 RDIR = get_rdir(run)
+shadow_config = get_shadow(run)
 
 shared_resources = run["shared_resources"]["policy"]
 exclude_from_shared = run["shared_resources"]["exclude"]
 logs = path_provider("logs/", RDIR, shared_resources, exclude_from_shared)
 benchmarks = path_provider("benchmarks/", RDIR, shared_resources, exclude_from_shared)
 resources = path_provider("resources/", RDIR, shared_resources, exclude_from_shared)
 
-CDIR = "" if run["shared_cutouts"] else RDIR
+cutout_dir = config["atlite"]["cutout_directory"]
+CDIR = join(cutout_dir, ("" if run["shared_cutouts"] else RDIR))
 RESULTS = "results/" + RDIR
 
 

config/config.default.yaml

@@ -30,6 +30,7 @@ run:
     policy: false
     exclude: []
   shared_cutouts: true
+  use_shadow_directory: true # Set to false if problems regarding missing directories occur
 
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#foresight
 foresight: overnight
@@ -132,6 +133,7 @@ electricity:
 
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#atlite
 atlite:
+  cutout_directory: cutouts
   default_cutout: europe-2013-sarah3-era5
   nprocesses: 4
   show_progress: false
@@ -956,6 +958,12 @@ solving:
       ipm_optimality_tolerance: 1e-4
       parallel: "on"
       random_seed: 123
+    highs-simplex:
+      solver: "simplex"
+      parallel: "on"
+      primal_feasibility_tolerance: 1e-5
+      dual_feasibility_tolerance: 1e-5
+      random_seed: 123
     gurobi-default:
       threads: 8
       method: 2 # barrier
@@ -1012,6 +1020,7 @@ solving:
     rtol: 0.01
 
   mem_mb: 30000 #memory in MB; 20 GB enough for 50+B+I+H2; 100 GB for 181+B+I+H2
+  memory_logging_frequency: 30 # in seconds
   runtime: 6h #runtime in humanfriendly style https://humanfriendly.readthedocs.io/en/latest/
 
 

config/test/config.electricity.yaml

@@ -80,10 +80,10 @@ lines:
 solving:
   solver:
     name: highs
-    options: highs-default
+    options: highs-simplex
 
   check_objective:
-    enable: false
+    enable: true
     expected_value: 3.8120188094e+07
 
 plotting:

config/test/config.myopic.yaml

@@ -85,7 +85,7 @@ industry:
 solving:
   solver:
     name: highs
-    options: highs-default
+    options: highs-simplex
   mem: 4000
 
 plotting:

config/test/config.overnight.yaml

@@ -79,12 +79,12 @@ industry:
 solving:
   solver:
     name: highs
-    options: highs-default
+    options: highs-simplex
   mem: 4000
 
   check_objective:
-    enable: false
-    expected_value: 7.0847670388e+08
+    enable: true
+    expected_value: 6.96e+08
 
 plotting:
   map:

config/test/config.perfect.yaml

@@ -84,12 +84,12 @@ industry:
 solving:
   solver:
     name: highs
-    options: highs-default
+    options: highs-simplex
   mem: 4000
 
   check_objective:
-    enable: false
-    expected_value: 1.4427662256e+10
+    enable: true
+    expected_value: 1.3778753459e+10
 
 plotting:
   map:

config/test/config.scenarios.yaml

@@ -58,4 +58,4 @@ renewable:
 solving:
   solver:
     name: highs
-    options: highs-default
+    options: highs-simplex

doc/configtables/atlite.csv

@@ -1,4 +1,5 @@
 ,Unit,Values,Description
+cutout_directory,--,str,"Directory to store cutouts."
 default_cutout,--,str,"Defines a default cutout."
 nprocesses,--,int,"Number of parallel processes in cutout preparation"
 show_progress,bool,true/false,"Whether progressbar for atlite conversion processes should be shown. False saves time."

doc/configtables/run.csv

@@ -9,3 +9,4 @@ shared_resources,,,
 -- policy,bool/str,,"Boolean switch to select whether resources should be shared across runs. If a string is passed, this is used as a subdirectory name for shared resources. If set to 'base', only resources before creating the elec.nc file are shared."
 -- exclude,str,"For the case shared_resources=base, specify additional files that should not be shared across runs."
 shared_cutouts,bool,"{true, false}","Switch to select whether cutouts should be shared across runs."
+use_shadow_directory,bool,"{true, false}","Set to ``true`` (default) if snakemake shadow directories (``shallow``) should be used. Set to ``false`` if problems occcur."

doc/configtables/solving.csv

@@ -38,3 +38,4 @@ solver,,,
 -- options,--,Key listed under ``solver_options``.,Link to specific parameter settings.
 solver_options,,dict,Dictionaries with solver-specific parameter settings.
 mem,MB,int,Estimated maximum memory requirement for solving networks.
+mem_logging_frequency,s,int,Interval in seconds at which memory usage is logged.

doc/release_notes.rst

@@ -15,6 +15,7 @@ Upcoming Release
 
 - ...
 
+- Bug fix: Added setting ``run: use_shadow_directory:`` (default: ``true``) which sets the ``shadow`` parameter of the snakemake workflow. Configuring to ``true`` sets snakemake ``shadow`` parameter to ``shalloow``, ``false`` to `Ǹone``. Should be set to ``false`` for those cases, where snakemake has an issue with finding missing input/output files in solving rules.
 
 PyPSA-Eur v2025.01.0 (24th January 2025)
 ========================================

envs/environment.yaml

@@ -55,15 +55,18 @@ dependencies:
 - rasterio
 
 # Development dependencies
+- jupyter
 - ipython
 - pre-commit
 - ruff
+- pylint
 
 - pip:
   - gurobipy
-  - highspy<1.8
+  - highspy
   - pyscipopt # See https://github.com/scipopt/PySCIPOpt/issues/944
   - tsam>=2.3.1
+  - entsoe-py
   - snakemake-storage-plugin-http
   - snakemake-executor-plugin-slurm
   - snakemake-executor-plugin-cluster-generic