CI-check for scenario management

.github/workflows/ci-linux.yml

@@ -3,10 +3,10 @@ name: CI-Linux
 on:
   push:
     branches:
-    - main
+    - scenario_management
   pull_request:
     branches:
-    - main
+    - scenario_management
   schedule:
   - cron: "0 5 * * TUE"
 
@@ -72,18 +72,47 @@ jobs:
     - name: Move test config
       run: |
         cd ../pypsa-earth
-        cp pypsa-kz-data/test/config_kz_test.yaml ./test/config.custom.yaml
+        cp pypsa-kz-data/test/config_kz_default_test.yaml ./test/config.landlock.yaml
 
     - name: Create test configs
       run: |
         cd ../pypsa-earth
         snakemake --cores all build_test_configs
 
-    - name: Test pypsa-kz-data workflow
+    - name: Clean scenarios folder
       run: |
         cd ../pypsa-earth
-        cp ./test/tmp/config.custom_tmp.yaml ./config.yaml
-        snakemake -j1 retrieve_databundle_light  # we first retrieve data
-        cp pypsa-kz-data/data/custom_powerplants.csv ./data/custom_powerplants.csv  # overwrite powerplantdata
-        snakemake -j1 solve_everything
+        rm -r pypsa-kz-data/scenarios/*
+
+    - name: Move test configs to scenario folder
+      run: |
+        cd ../pypsa-earth
+        cp pypsa-kz-data/test/config_kz_test1.yaml pypsa-kz-data/scenarios/config.kz_2013.yaml
+        cp ./test/tmp/config.landlock_tmp.yaml pypsa-kz-data/config.kz_default.yaml
+
+    - name : Change Snakefile lines
+      run: |
+        cd ../pypsa-earth
+        sed -i 's\os.system("snakemake -j all solve_all_networks --rerun-incomplete")\os.system(f"snakemake -j1 networks/{wildcards.scenario_name}/base.nc")\g' Snakefile
+        sed -i 's,os.system("snakemake -j1 make_statistics --force"),os.system("cp pypsa-kz-data/data/custom_powerplants.csv data/custom_powerplants.csv")\n\tos.system("snakemake -j1 solve_all_networks --rerun-incomplete"),g' Snakefile
+        cat Snakefile
+
+    - name: Replace Default config
+      run: |
+        cd ../pypsa-earth
+        cp pypsa-kz-data/config.kz_default.yaml config.default.yaml
+        cp config.default.yaml config.yaml
+
+    - name: Prepare KZ-scenarios
+      run: |
+        cd ../pypsa-earth
+        snakemake -j1 prepare_kz_scenarios
+        cp -r pypsa-kz-data/test/gadm ./data/gadm
+        find data -type f
+        cat configs/scenarios/config.kz_2013.yaml
+
+    - name: Run all scenarios
+      run: |
+        cd ../pypsa-earth
+        snakemake -j1 run_all_scenarios
 

README.md

@@ -14,7 +14,7 @@ Agora Energiewende aims to model the Kazakh power system, incorporating a substa
 
 ## Development status: Active and Stable
 
-[![CI-Linux](https://github.com/pypsa-meets-earth/pypsa-kz-data/actions/workflows/ci-linux.yml/badge.svg?branch=main&event=push)](https://github.com/pypsa-meets-earth/pypsa-kz-data/actions/workflows/ci-linux.yml)
+[![CI-Linux](https://github.com/pypsa-meets-earth/pypsa-kz-data/actions/workflows/ci-linux.yml/badge.svg?branch=main)](https://github.com/pypsa-meets-earth/pypsa-kz-data/actions/workflows/ci-linux.yml)
 ![Size](https://img.shields.io/github/repo-size/pypsa-meets-earth/pypsa-kz-data?label=Repo%20size)
 [![License: AGPL v3](https://img.shields.io/badge/License-AGPLv3-blue.svg)](https://www.gnu.org/licenses/agpl-3.0)
 [![REUSE status](https://api.reuse.software/badge/github.com/pypsa-meets-earth/pypsa-kz-data)](https://api.reuse.software/info/github.com/pypsa-meets-earth/pypsa-kz-data)

test/config_kz_test.yaml → test/config_kz_default_test.yaml

@@ -6,14 +6,14 @@
 tutorial: false
 
 run:
-  name: "kz_test" # use this to keep track of runs with different settings
+  name: "kz_2013_default" # use this to keep track of runs with different settings
   shared_cutouts: true  # set to true to share the default cutout(s) across runs
                         # Note: value false requires build_cutout to be enabled
 
 scenario:
   ll: ['v1.0']
-  clusters: [14]
-  opts: [RES0.2-1H]
+  clusters: [5]
+  opts: [BAU-RES0.2-6H]
 
 
 countries: ["KZ"]
@@ -38,13 +38,13 @@ cluster_options:
   simplify_network:
     p_threshold_drop_isolated: 10 # [MW] isolated buses are being discarded if bus mean power is below the specified threshold
 
-  alternative_clustering: true #false  # "False" use Voronoi shapes, "True" use GADM shapes
+  alternative_clustering: false  # "False" use Voronoi shapes, "True" use GADM shapes
 
 
 build_shape_options:
   nprocesses: 1  # number of processes to be used in build_shapes
   nchunks: 3  # number of data chuncks for build_shapes. If not specified or smaller than nprocesses, specified as nprocesses
-
+  gadm_layer_id: 0
 
 load_options:
   rescale_demand: true

test/config_kz_test1.yaml

@@ -0,0 +1,117 @@
+# SPDX-FileCopyrightText:  pypsa-kz-data authors
+#
+# SPDX-License-Identifier: CC0-1.0
+
+
+tutorial: false
+
+run:
+  name: "kz_2013" # use this to keep track of runs with different settings
+  shared_cutouts: true  # set to true to share the default cutout(s) across runs
+                        # Note: value false requires build_cutout to be enabled
+
+scenario:
+  ll: ['v1.0']
+  clusters: [5]
+  opts: [6H]
+
+
+countries: ["KZ"]
+
+snapshots:
+  start: "2013-01-01"
+  end: "2013-01-02"
+
+
+enable:
+  build_natura_raster: false  # If True, than an exclusion raster will be build
+
+
+retrieve_databundle: # required to be "false" for nice CI test output
+  show_progress: true #false # show (true) or do not show (false) the progress bar in retrieve_databundle while downloading data
+
+
+custom_rules: ["pypsa-kz-data/kz.smk"]
+
+
+cluster_options:
+  simplify_network:
+    p_threshold_drop_isolated: 10 # [MW] isolated buses are being discarded if bus mean power is below the specified threshold
+
+  alternative_clustering: false  # "False" use Voronoi shapes, "True" use GADM shapes
+
+
+build_shape_options:
+  nprocesses: 1  # number of processes to be used in build_shapes
+  nchunks: 3  # number of data chuncks for build_shapes. If not specified or smaller than nprocesses, specified as nprocesses
+  gadm_layer_id: 0
+
+load_options:
+  rescale_demand: true
+  external_loads: true # true - adds external load buses based on data/custom_data/electricity_exp-imp.csv
+
+electricity:
+  base_voltage: 500.
+  voltages: [110., 220., 500.]
+  co2limit: 7.75e+11  # European default, 0.05 * 3.1e9*0.5, needs to be adjusted for Africa
+  co2base: 3.17e+8  # Kazakhstan default for 1990
+
+
+  extendable_carriers:
+    Generator: []  
+
+
+  custom_powerplants: replace #merge #replace # "false" - no custom powerplants, "merge" use both open source and custom powerplants, "replace" use only custom powerplants
+
+
+  estimate_renewable_capacities:
+    stats: False #'irena'  # False, = greenfield expansion, 'irena' uses IRENA stats to add expansion limits
+
+
+lines:
+  types:
+    220.: "490-AL1/64-ST1A 220.0"             #"Al/St 240/40 2-bundle 220.0"
+    110.: "243-AL1/39-ST1A 110.0"             #"Al/St 240/40 3-bundle 300.0"
+    500.: "Al/St 240/40 4-bundle 380.0"       #"Al/St 560/50 4-bundle 750.0"       #
+
+  modify_lines:
+    limit_line_capacities: true # limiting line capacities
+    bus_of_interest: ["KZ.1_1_AC", "KZ.9_1_AC"] # buses of interest whose lines are limited
+    max_limit: 700.0 # maximum limit for s_nom
+
+
+atlite:
+  nprocesses: 4
+  cutouts:
+    # geographical bounds automatically determined from countries input
+    cutout-2013-era5-tutorial:
+      module: era5
+      dx: 0.3  # cutout resolution
+      dy: 0.3  # cutout resolution
+
+
+renewable:
+  onwind:
+    cutout: cutout-2013-era5-tutorial
+
+  offwind-ac:
+    cutout: cutout-2013-era5-tutorial
+
+  offwind-dc:
+    cutout: cutout-2013-era5-tutorial
+
+  solar:
+    cutout: cutout-2013-era5-tutorial
+
+  hydro:
+    cutout: cutout-2013-era5-tutorial
+    extendable: true
+    normalization:
+      method: eia #hydro_capacities  # 'hydro_capacities' to rescale country hydro production by using hydro_capacities, 'eia' to rescale by eia data, false for no rescaling
+      year: 2013  # (optional) year of statistics used to rescale the runoff time series. When not provided, the weather year of the snapshots is used
+
+
+costs:
+  year: 2020
+  fill_values:
+    discount rate: 0.15