test.yaml

# SPDX-FileCopyrightText: : 2021-2024 The PyPSA-Eur Authors
#
# SPDX-License-Identifier: CC0-1.0

name: Test workflows

on:
  push:
    branches:
    - master
  pull_request:
    branches:
    - master
  schedule:
  - cron: "0 5 * * *"

# Cancel any in-progress runs when a new run is triggered
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

env:
  ENV_FILE: envs/environment.yaml
  ENV_FIXED_FILE: envs/environment.fixed.yaml

jobs:
  run-tests:
    name: OS
    strategy:
      fail-fast: false
      matrix:
        os:
        - macos
        - windows
        - ubuntu
    runs-on: ${{ matrix.os }}-latest

    defaults:
      run:
        shell: bash -l {0}

    steps:
    - uses: actions/checkout@v4

    - name: Setup secrets & cache dates
      run: |
        echo -ne "url: ${CDSAPI_URL}\nkey: ${CDSAPI_TOKEN}\n" > ~/.cdsapirc
        echo "week=$(date +'%Y%U')" >> $GITHUB_ENV # data and cutouts
        echo "today=$(/bin/date -u '+%Y%m%d')" >> $GITHUB_ENV # env

    - uses: actions/cache@v4
      with:
        path: |
          data
          cutouts
        key: data-cutouts-${{ env.week }}

    - uses: conda-incubator/setup-miniconda@v3
      with:
        activate-environment: pypsa-eur

    - name: Cache Conda env
      uses: actions/cache@v4
      with:
        path: ${{ env.CONDA }}/envs
        key: conda-${{ runner.os }}-${{ runner.arch }}-${{ env.today }}-${{ hashFiles(format('{0}', env.ENV_FILE)) }}
      id: cache-env

    - name: Update environment
      if: steps.cache-env.outputs.cache-hit != 'true'
      run: conda env update -n pypsa-eur -f ${{ env.ENV_FILE }}

    - name: Log env diff to environment.fixed.yaml
      if: ${{ matrix.os == 'ubuntu' }}
      run: |
        # Get fixed environment of current env
        conda env export --name ${{ github.event.repository.name }} --no-builds | sed 's/^name: ${{ github.event.repository.name }}$/name: ${{ github.event.repository.name }}-fixed/' > current-env.yaml

        # Add SPDX header
        SPDX_HEADER="# SPDX-FileCopyrightText: : 2017-2024 The PyPSA-Eur Authors\n# SPDX-License-Identifier: CC0-1.0\n"
        echo -e "$SPDX_HEADER" | cat - current-env.yaml > temp && mv temp current-env.yaml

        # Format with pre-commit (it differs from the conda output)
        pip install pre-commit
        pre-commit run --files current-env.yaml || true

        # Get diff
        diff ${{ env.ENV_FIXED_FILE }} current-env.yaml > diff.txt || true

        # Format
        { echo -e "**Environment diff**\n\`\`\` diff\n"; cat diff.txt; echo -e "\n\`\`\`"; } > temp && mv temp diff.txt

        # Write to summary
        cat diff.txt >> $GITHUB_STEP_SUMMARY
      shell: bash

    - name: Run snakemake test workflows
      run: |
        make test

    - name: Upload artifacts
      uses: actions/upload-artifact@v3.2.1-node20
      with:
        name: logs-${{ matrix.inhouse }}
        path: |
          logs
          .snakemake/log
        retention-days: 3

  run-tests-on-dev-deps:
    name: Inhouse
    strategy:
      fail-fast: false
      matrix:
        inhouse:
        - pypsa
        - atlite
        - powerplantmatching
        - linopy
    runs-on: ubuntu-latest

    defaults:
      run:
        shell: bash -l {0}

    steps:
    - uses: actions/checkout@v4

    # Only run checks if package is not pinned
    - name: Check if inhouse package is pinned
      run: |
        grep_line=$(grep -- '- ${{ matrix.inhouse }}' ${{ env.ENV_FILE }})
        if [[ $grep_line == *"<"* || $grep_line == *"=="* ]]; then
            echo "pinned=true" >> $GITHUB_ENV
        else
            echo "pinned=false" >> $GITHUB_ENV
        fi

    - name: Setup secrets & cache dates
      if: env.pinned == 'false'
      run: |
        echo -ne "url: ${CDSAPI_URL}\nkey: ${CDSAPI_TOKEN}\n" > ~/.cdsapirc
        echo "week=$(date +'%Y%U')" >> $GITHUB_ENV # data and cutouts
        echo "today=$(/bin/date -u '+%Y%m%d')" >> $GITHUB_ENV # env

    - uses: actions/cache@v4
      if: env.pinned == 'false'
      with:
        path: |
          data
          cutouts
        key: data-cutouts-${{ env.week }}

    - uses: conda-incubator/setup-miniconda@v3
      if: env.pinned == 'false'
      with:
        activate-environment: pypsa-eur

    - name: Cache Conda env
      if: env.pinned == 'false'
      uses: actions/cache@v4
      with:
        path: ${{ env.CONDA }}/envs
        key: conda-${{ runner.os }}-${{ runner.arch }}-${{  matrix.inhouse  }}-${{ env.today }}-${{ hashFiles(format('{0}', env.ENV_FILE)) }}
      id: cache-env

    - name: Update environment
      if: env.pinned == 'false' && steps.cache-env.outputs.cache-hit != 'true'
      run: conda env update -n pypsa-eur -f ${{ env.ENV_FILE }}

    - name: Install inhouse packages from master
      if: env.pinned == 'false'
      run: |
        python -m pip install git+https://github.com/PyPSA/${{ matrix.inhouse }}.git@master

    - name: Run snakemake test workflows
      if: env.pinned == 'false'
      run: |
        make test

    - name: Upload artifacts
      if: env.pinned == 'false'
      uses: actions/upload-artifact@v4
      with:
        name: logs-${{ matrix.inhouse }}
        path: |
          logs
          .snakemake/log
        retention-days: 3