Split the github workflow in CI and CD (#1063)

* First version

* Deleted the old CI

* Do not run on draft PRs

* Deleted comment

* Added --exitfirst

* Commented out if for testing the workflow

* Moved the pull_request tag from CI to CD

* Changing on push and pull_request for CI and CD

* More specific concurrency

* Debugging why CD is not triggered

* Moved if clause

* Bug fix

* Changed flags for code cov

* Added -n auto from recent PR

* Installing pytest cov with the dev option

* Updated testing and cov options

* lfs is not used

* Revert to pre-commit hooks as a separate job

* First try of the codecov.yml

* Format fix

* No tokes was found, is this due to the codecov.yml?

* The token was still not found

* Before this commit a token was found

* Test doc bulding

* Docs workflow worked, moved it to CD

* More detailed caching key

* Run fast tests for pushes and fast+slow for PRs

* Testing double exec

* Revert since the last commit is only executed when the PR is merged

* Add fallback keys if full key is not available

* Do not run the slow tests in CI

* Fixed typo

* Add a workflow for manual selection of markers

* Added coverage report for fast CI back in

---------

Co-authored-by: Sebastian Bischoff <[email protected]>

.github/codecov.yml

@@ -0,0 +1,28 @@
+# The official docs
+# https://docs.codecov.io/docs/codecov-yaml
+# https://docs.codecov.com/docs/common-recipe-list
+
+# Check if this file is valid
+# cd PATH_TO/sbi/.github
+# curl -X POST --data-binary @codecov.yml https://codecov.io/validate
+
+ignore:
+  - "sbi/examples"
+
+coverage:
+  status:
+    project:
+      default:
+        target: 70% # the required coverage value
+        threshold: 2% # allow the coverage to drop by X%, and posting a success status
+        if_ci_failed: error # will set the status to success only if the CI is successful, alternative: success
+    patch: # about the individual commit
+      default:
+        target: 50% # minimum coverage ratio that the commit must meet to be considered a success
+        threshold: 2% # allow the coverage to drop by X%, and posting a success status
+        if_ci_failed: error # will set the status to success only if the CI is successful, alternative: success
+
+comment:
+  layout: "diff, flags, files"
+  behavior: default # update if exists, otherwise post new
+  require_changes: false # if true, only post the comment if coverage changes

.github/workflows/cd.yml

@@ -0,0 +1,78 @@
+name: Continuous Deployment
+
+on:
+  push:
+    branches: [main]
+  workflow_dispatch:
+
+defaults:
+  run:
+    shell: bash
+
+concurrency:
+  group:  ${{ github.workflow }}-${{ github.head_ref || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  pre-commit:
+    name: ruff and hooks.
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.8'
+      - uses: pre-commit/[email protected]
+        with:
+          extra_args: --all-files --show-diff-on-failure
+
+  cd:
+    name: CD
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+        lfs: false
+
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.8'
+
+    - name: Cache dependency
+      id: cache-dependencies
+      uses: actions/cache@v4
+      with:
+        path: ~/.cache/pip
+        key: ${{ runner.os }}-pip
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -e .[dev]
+
+    - name: Check types with pyright
+      run: |
+        pyright sbi
+
+    - name: Run the fast and the slow CPU tests with coverage
+      run: |
+        pytest -v -x -n auto -m "not gpu" --cov=sbi --cov-report=xml tests/
+
+    - name: Upload coverage to Codecov
+      uses: codecov/codecov-action@v4-beta
+      with:
+        env_vars: OS,PYTHON
+        file: ./coverage.xml
+        flags: unittests
+        name: codecov-sbi-all-cpu
+        token: ${{ secrets.CODECOV_TOKEN }}
+
+    - name: Check doc building
+      run: |
+        jupyter nbconvert --to markdown tutorials/*.ipynb --output-dir docs/tutorial/
+        jupyter nbconvert --to markdown examples/*.ipynb --output-dir docs/examples/
+        mkdocs build -f docs/mkdocs.yml --site-dir site

.github/workflows/ci.yml

@@ -0,0 +1,81 @@
+name: Continuous Integration
+
+on: [pull_request, workflow_dispatch]
+
+defaults:
+  run:
+    shell: bash
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  pre-commit:
+    name: ruff and hooks.
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.8'
+      - uses: pre-commit/[email protected]
+        with:
+          extra_args: --all-files --show-diff-on-failure
+
+  ci:
+    name: CI
+    runs-on: ubuntu-latest
+    if: |
+      github.event_name == 'push' ||
+      (github.event_name == 'pull_request' && github.event.pull_request.draft == false)
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ['3.8']
+        torch-version: ['1.11', '2.2']
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+        lfs: false
+
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Cache dependency
+      id: cache-dependencies
+      uses: actions/cache@v4
+      with:
+        path: ~/.cache/pip
+        key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ matrix.torch-version }}$
+        restore-keys: |
+          ${{ runner.os }}-pip-${{ matrix.python-version }}-
+          ${{ runner.os }}-pip-
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install torch==${{ matrix.torch-version }} --extra-index-url https://download.pytorch.org/whl/cpu
+        pip install -e .[dev]
+
+    - name: Check types with pyright
+      run: |
+        pyright sbi
+
+    - name: Run the fast CPU tests with coverage
+      run: |
+        pytest -v -x -n auto -m "not slow and not gpu" --cov=sbi --cov-report=xml tests/
+
+    - name: Upload coverage to Codecov
+      uses: codecov/codecov-action@v4
+      with:
+        env_vars: OS,PYTHON
+        file: ./coverage.xml
+        flags: unittests
+        name: codecov-sbi-fast-cpu
+        token: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/manual_test.yml

@@ -0,0 +1,65 @@
+name: Manual-Test
+
+on:
+  workflow_dispatch:
+    inputs:
+      pytest-marker:
+        description: "Combination of markers to restrict the tests, use '' to run all tests."
+        type: choice
+        options:
+          - 'not slow and not gpu'
+          - 'not gpu'
+          - 'not slow'
+          - ''
+        default: ''
+        required: true
+
+defaults:
+  run:
+    shell: bash
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    name: manual-test
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ['3.8']
+        torch-version: ['1.11', '2.2']
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+        lfs: false
+
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Cache dependency
+      id: cache-dependencies
+      uses: actions/cache@v4
+      with:
+        path: ~/.cache/pip
+        key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ matrix.torch-version }}$
+        restore-keys: |
+          ${{ runner.os }}-pip-${{ matrix.python-version }}-
+          ${{ runner.os }}-pip-
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install torch==${{ matrix.torch-version }} --extra-index-url https://download.pytorch.org/whl/cpu
+        pip install -e .[dev]
+
+    - name: Run the selected tests without coverage
+      run: |
+        pytest -v -x -m ${{ inputs.pytest-marker }} tests/

pyproject.toml

@@ -62,6 +62,7 @@ dev = [
     "ruff>=0.3.3",
     # Test
     "pytest",
+    "pytest-cov",
     "pytest-xdist",
     "torchtestcase",
 ]