test: fix and update test workflows (#1296)

* test: fix and update test workflows

* resolve conflict

* bring back test.sh

.github/workflows/test.yaml

@@ -0,0 +1,112 @@
+# SPDX-FileCopyrightText: : 2021-2024 The PyPSA-Eur Authors
+#
+# SPDX-License-Identifier: CC0-1.0
+
+name: Test workflows
+
+on:
+  push:
+    branches:
+    - master
+  pull_request:
+    branches:
+    - master
+  schedule:
+  - cron: "0 5 * * TUE"
+
+# Cancel any in-progress runs when a new run is triggered
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  run:
+    name: Run
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+        - macos
+        - windows
+        - ubuntu
+        inhouse:
+        - stable-inhouse-deps
+        - dev-inhouse-deps
+        exclude:
+        - os: macos
+          inhouse: dev-inhouse-deps
+        - os: windows
+          inhouse: dev-inhouse-deps
+    runs-on: ${{ matrix.os }}-latest
+
+    defaults:
+      run:
+        shell: bash -l {0}
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Setup secrets
+      run: |
+        echo -ne "url: ${CDSAPI_URL}\nkey: ${CDSAPI_TOKEN}\n" > ~/.cdsapirc
+
+    - name: Set cache dates
+      run: |
+        echo "week=$(date +'%Y%U')" >> $GITHUB_ENV # data and cutouts
+        echo "today=$(/bin/date -u '+%Y%m%d')" >> $GITHUB_ENV # env
+
+    - name: Cache data and cutouts folders
+      uses: actions/cache@v4
+      with:
+        path: |
+          data
+          cutouts
+        key: data-cutouts-${{ env.week }}
+
+    - name: Setup Conda
+      uses: conda-incubator/setup-miniconda@v3
+      with:
+        activate-environment: pypsa-eur
+
+    - name: Cache Conda env
+      uses: actions/cache@v3
+      with:
+        path: ${{ env.CONDA }}/envs
+        key: conda-${{ runner.os }}--${{ runner.arch }}--${{ env.today }}-${{ hashFiles('envs/environment.yaml') }}
+      id: cache-env
+
+    - name: Update environment
+      if: steps.cache-env.outputs.cache-hit != 'true' && matrix.os != 'macos'
+      run: conda env update -n pypsa-eur -f envs/environment.yaml
+
+    # Temporary fix for MacOS, since highspy with pypsa can not be resolved
+    - name: Update environment (macos specific)
+      if: steps.cache-env.outputs.cache-hit != 'true' && matrix.os == 'macos'
+      run: |
+        sed -i '' '/- pypsa/d' envs/environment.yaml # Remove pypsa from environment list
+        conda env update -n pypsa-eur -f envs/environment.yaml
+        pip install highspy
+        conda install -c conda-forge pypsa
+
+    - name: Install inhouse packages from master
+      if: matrix.inhouse == 'dev-inhouse-deps'
+      run: |
+        python -m pip install uv
+        uv pip install git+https://github.com/PyPSA/pypsa.git@master
+        uv pip install git+https://github.com/PyPSA/atlite.git@master
+        uv pip install git+https://github.com/PyPSA/powerplantmatching.git@master
+        uv pip install git+https://github.com/PyPSA/linopy.git@master
+
+    - name: Run snakemake test workflows
+      run: |
+        chmod +x test.sh
+        ./test.sh
+
+    - name: Upload artifacts
+      if: matrix.os == 'ubuntu' && matrix.inhouse == 'stable-inhouse-deps'
+      uses: actions/[email protected]
+      with:
+        name: resources-results
+        path: |
+          results
+        retention-days: 7

README.md

@@ -4,7 +4,7 @@ SPDX-License-Identifier: CC-BY-4.0
 -->
 
 ![GitHub release (latest by date including pre-releases)](https://img.shields.io/github/v/release/pypsa/pypsa-eur?include_prereleases)
-[![Build Status](https://github.com/pypsa/pypsa-eur/actions/workflows/ci.yaml/badge.svg)](https://github.com/PyPSA/pypsa-eur/actions)
+[![Test workflows](https://github.com/pypsa/pypsa-eur/actions/workflows/test.yaml/badge.svg)](https://github.com/pypsa/pypsa-eur/actions/workflows/test.yaml)
 [![Documentation](https://readthedocs.org/projects/pypsa-eur/badge/?version=latest)](https://pypsa-eur.readthedocs.io/en/latest/?badge=latest)
 ![Size](https://img.shields.io/github/repo-size/pypsa/pypsa-eur)
 [![Zenodo PyPSA-Eur](https://zenodo.org/badge/DOI/10.5281/zenodo.3520874.svg)](https://doi.org/10.5281/zenodo.3520874)

doc/index.rst

@@ -10,7 +10,7 @@ PyPSA-Eur: A Sector-Coupled Open Optimisation Model of the European Energy Syste
 .. image:: https://img.shields.io/github/v/release/pypsa/pypsa-eur?include_prereleases
     :alt: GitHub release (latest by date including pre-releases)
 
-.. image:: https://github.com/pypsa/pypsa-eur/actions/workflows/ci.yaml/badge.svg
+.. image:: https://github.com/pypsa/pypsa-eur/actions/workflows/test.yaml/badge.svg
     :target: https://github.com/PyPSA/pypsa-eur/actions
 
 .. image:: https://readthedocs.org/projects/pypsa-eur/badge/?version=latest

envs/environment.yaml

@@ -15,7 +15,7 @@ dependencies:
 - linopy
 - dask
 
-  # Dependencies of the workflow itself
+# Dependencies of the workflow itself
 - xlrd
 - openpyxl!=3.1.1
 - seaborn
@@ -49,15 +49,14 @@ dependencies:
 - pre-commit
 - geojson
 
-  # Keep in conda environment when calling ipython
+# Keep in conda environment when calling ipython
 - ipython
 
-  # GIS dependencies:
+# GIS dependencies:
 - cartopy
 - descartes
 - rasterio!=1.2.10
 
-
 - pip:
   - tsam>=2.3.1
   - snakemake-storage-plugin-http