Merge remote-tracking branch 'origin/master' into circuit-extensions

.github/workflows/code_check.yml

@@ -1,20 +1,17 @@
 name: Code Checking
 
-on: 
-  push:
-    branches-ignore:
-      - master
-    paths:
-      - '**/*.py'
+on:
   pull_request:
     branches:
       - master
     paths:
+      - ! 'resources/retinal-rl.def'
       - '**/*.py'
+  workflow_dispatch:
 
 env:
-  singularity_image: oras://ghcr.io/berenslab/retinal-rl:singularity-image-latest
-  sif_file: retinal-rl_singularity-image-latest.sif
+  singularity_image: oras://ghcr.io/berenslab/retinal-rl:singularity-image
+  sif_file: retinal-rl_singularity-image.sif
 
 jobs:
   check:
@@ -31,22 +28,15 @@ jobs:
       uses: eWaterCycle/setup-apptainer@v2
       with:
         apptainer-version: 1.3.0
+
     - name: Cache Singularity Image
       id: cache-singularity
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: ${{ env.sif_file }}
         key: ${{ runner.os }}-singularity-${{ hashFiles('~/resources/retinal-rl.def') }}
         restore-keys: |
           ${{ runner.os }}-singularity-${{ hashFiles('~/resources/retinal-rl.def') }}
-          ${{ runner.os }}-singularity-
-    - name: Pull Singularity container
-      if: steps.cache-singularity.outputs.cache-hit != 'true'
-      run: |
-        singularity registry login --username ${{ github.actor }} --password ${{ secrets.GITHUB_TOKEN }} oras://ghcr.io
-        singularity pull ${{ env.sif_file }} ${{ env.singularity_image }}
 
     - name: Run Pylint
-      run: |
-        singularity exec ${{ env.sif_file }} \
-        pylint $(git diff --name-only origin/master...HEAD -- '*.py')
+      run: bash tests/ci/pylint.sh ${{ env.sif_file }}

.github/workflows/config_scan.yml

@@ -1,9 +1,13 @@
 name: Scan Configs
-on: [pull_request,workflow_dispatch]
+on: 
+  pull_request:
+    paths-ignore: 
+      - 'resources/retinal-rl.def'
+  workflow_dispatch:
 
 env:
-  singularity_image: oras://ghcr.io/berenslab/retinal-rl:singularity-image-latest
-  sif_file: retinal-rl_singularity-image-latest.sif
+  singularity_image: oras://ghcr.io/berenslab/retinal-rl:singularity-image
+  sif_file: retinal-rl_singularity-image.sif
 
 jobs:
   scan:
@@ -16,30 +20,15 @@ jobs:
         apptainer-version: 1.3.0
     - name: Cache Singularity Image
       id: cache-singularity
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: ${{ env.sif_file }}
         key: ${{ runner.os }}-singularity-${{ hashFiles('~/resources/retinal-rl.def') }}
         restore-keys: |
           ${{ runner.os }}-singularity-${{ hashFiles('~/resources/retinal-rl.def') }}
-          ${{ runner.os }}-singularity-
-    - name: Pull Singularity container
-      if: steps.cache-singularity.outputs.cache-hit != 'true'
-      run: |
-        singularity registry login --username ${{ github.actor }} --password ${{ secrets.GITHUB_TOKEN }} oras://ghcr.io
-        singularity pull ${{ env.sif_file }} ${{ env.singularity_image }}
 
     - name: Prepare experiment files
-      run: |
-        cp -r resources/config_templates/* config/
-        for file in config/user/experiment/*.yaml; do
-          filename=$(basename "$file" .yaml)
-          echo "$filename" >> experiments.txt
-        done
+      run: bash tests/ci/copy_configs.sh ${{ env.sif_file }}
 
-    - name: Scan classification configs
-      run: |
-        while IFS= read -r experiment; do
-          singularity exec ${{ env.sif_file }} \
-          python main.py +experiment="$experiment" command=scan system.device=cpu
-        done < experiments.txt
+    - name: Scan configs
+      run: bash tests/ci/scan_configs.sh ${{ env.sif_file }}

.github/workflows/container_build.yml

@@ -1,37 +1,74 @@
 name: Build Singularity Container
 
 on: 
-  schedule:
-    - cron: '0 2 1 * *'
-  push:
-    branches-ignore:
-      - master
-    paths:
-      - 'resources/retinal-rl.def'
   pull_request:
     branches:
       - master
     paths:
       - 'resources/retinal-rl.def'
+  workflow_call:
+    inputs:
+      deploy:
+        type: boolean
+        required: false
+        default: true
+  workflow_dispatch:
+    inputs:
+      deploy:
+        type: boolean
+        required: false
+        default: true
+
+env:
+  singularity_image: oras://ghcr.io/berenslab/retinal-rl:singularity-image
+  sif_file: retinal-rl_singularity-image.sif
 
 jobs:
   singularity-build:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v4
+    - name: Checkout code
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+    - name: Fetch all branches
+      run: git fetch --all
     - uses: eWaterCycle/setup-apptainer@v2
       with:
         apptainer-version: 1.3.0
+
+    - name: Evaluate whether build should be deployed
+      id: setup
+      run: |
+        if [[ "${{ inputs.deploy }}" != "" ]]; then
+          echo "deploy=${{ inputs.deploy }}" >> $GITHUB_OUTPUT
+        else
+          echo "deploy=false" >> $GITHUB_OUTPUT
+        fi
+
+    - name: Cache Singularity Image
+      id: cache-singularity
+      uses: actions/cache@v4
+      with:
+        path: ${{ env.sif_file }}
+        key: ${{ runner.os }}-singularity-${{ hashFiles('~/resources/retinal-rl.def') }}
+        restore-keys: |
+          ${{ runner.os }}-singularity-${{ hashFiles('~/resources/retinal-rl.def') }}
 
     - name: Build Singularity container
-      run: apptainer build retinal-rl.sif resources/retinal-rl.def
+      if: steps.cache-singularity.outputs.cache-hit != 'true'
+      run: apptainer build ${{ env.sif_file }} resources/retinal-rl.def
 
-    - name: Scan classification config / ensure minimal functionality
+    - name: Scan configs
       run: |
-        cp -r resources/config_templates/* config/
-        singularity exec retinal-rl.sif python main.py -m +experiment=classification command=scan system.device=cpu
+        bash tests/ci/copy_configs.sh ${{ env.sif_file }}
+        bash tests/ci/scan_configs.sh ${{ env.sif_file }}
+
+    - name: Run code check
+      run: bash tests/ci/pylint.sh ${{ env.sif_file }}
 
-    - name: Push to ghcr.io
+    - name: Deployment / Push to ghcr.io
+      if: steps.setup.outputs.deploy == 'true'
       run: |
-        singularity registry login --username ${{ github.actor }} --password ${{ secrets.GITHUB_TOKEN }} oras://ghcr.io
-        singularity push retinal-rl.sif oras://ghcr.io/berenslab/retinal-rl:singularity-image-latest
+        apptainer registry login --username ${{ github.actor }} --password ${{ secrets.GITHUB_TOKEN }} oras://ghcr.io
+        apptainer push ${{ env.sif_file }} oras://ghcr.io/berenslab/retinal-rl:singularity-image

.github/workflows/update_cache.yml

@@ -0,0 +1,42 @@
+name: Update Master Cache
+
+on: 
+  schedule:
+    - cron: '0 2 * * 0' # As cache is emptied if not accessed for 7 days, check after 7 days whether container has changed
+  workflow_dispatch:
+  push:
+    branches:
+      - master
+    paths:
+      - 'resources/retinal-rl.def'
+  workflow_call:
+
+env:
+  singularity_image: oras://ghcr.io/berenslab/retinal-rl:singularity-image
+  sif_file: retinal-rl_singularity-image.sif
+
+jobs:
+  conditional-build:
+    uses: ./.github/workflows/container_build.yml
+    with:
+      deploy: ${{ github.event_name == 'push' }}
+  update-cache:
+    needs: conditional-build
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Setup Apptainer
+      uses: eWaterCycle/setup-apptainer@v2
+      with:
+        apptainer-version: 1.3.0
+    - name: Cache Singularity Image
+      id: cache-singularity
+      uses: actions/cache@v4
+      with:
+        path: ${{ env.sif_file }}
+        key: ${{ runner.os }}-singularity-${{ hashFiles('~/resources/retinal-rl.def') }}
+        restore-keys: |
+          ${{ runner.os }}-singularity-${{ hashFiles('~/resources/retinal-rl.def') }}
+    - name: Pull Singularity container
+      if: steps.cache-singularity.outputs.cache-hit != 'true'
+      run: apptainer pull ${{ env.sif_file }} ${{ env.singularity_image }}

.pylintrc

@@ -1,2 +1,15 @@
 [MASTER]
-init-hook='import sys; sys.path.append(".")'
+init-hook='import sys; sys.path.append(".")'
+
+[MESSAGES CONTROL]
+
+# Enable the message, report, category or checker with the given id(s). You can
+# either give multiple identifier separated by comma (,) or put this option
+# multiple time.
+#enable=
+
+# Disable the message, report, category or checker with the given id(s). You
+# can either give multiple identifier separated by comma (,) or put this option
+# multiple time (only on the command line, not in the configuration file where
+# it should appear only once).
+disable=missing-function-docstring,missing-module-docstring

README.md

@@ -1,60 +1,54 @@
+# Retinal RL
+
 ## Setting up the development environment
 
-Unfortunately, putting together a unified build scheme has proven challenging, because our different compute resources rely on different containerization schemes (i.e. bespoke docker vs apptainer), and subtle bugs have emerged that only effect one build system or the other. We maintain a `Dockerfile` for building the docker image and the `retinal_rl.def` file to build the `apptainer` image, and we've also had success building with `conda` on bare metal.
+We provide a singularity / apptainer container which should always be up to date and allow to run the code immediately. You do not need to build it yourself (but can, of course), you can just pull it!
 
-### Apptainer
+### Install Singularity / Apptainer
 
-The `apptainer` image is self-contained, and building it should immediately allow running the relevant scripts in `retinal-rl`, by prefixing them with `apptainer exec [image]`. The versions of most pip packages are floating, but we have an `environment.yaml` file from a working `apptainer` build.
+First you need to install [apptainer](https://github.com/apptainer/apptainer/) (or singularity) in order to run code.
 
-### Conda
+### Get the container
 
-Here are the steps to get a `retinal-rl` environment setup in `conda`, which should work on bare metal. First [install anaconda or miniconda](https://docs.anaconda.com/anaconda/install/index.html), and then create the environment
-``` bash
-conda create --name retinal-rl pip
-conda activate retinal-rl
-```
-I'm using `miniconda`, so some of the following commands might be redundant if you're using `anaconda`.
+Once you have apptainer installed, you can simply pull the container
 
-Next, we install `pytorch`
-```bash
-conda install pytorch torchvision torchaudio cudatoolkit=11.6 -c pytorch -c conda-forge
-```
-and then install `sample-factory` and `vizdoom`
 ```bash
-pip install sample-factory
-pip install vizdoom
+apptainer pull retinal-rl.sif oras://ghcr.io/berenslab/retinal-rl:singularity-image
 ```
-Note, you may require `sample-factory=1.121.4` on a server. You may also want to downgrade the `gym` library with `pip install gym==0.25.2`
 
-For `vizdoom`, `pip install` can sometimes fail when run inside a `conda` environment. In this case the solution is to build `vizdoom` directly by running
-```bash
-conda install -c conda-forge boost cmake gtk2 sdl2
-git clone https://github.com/mwydmuch/ViZDoom.git --recurse-submodules
-cd ViZDoom
-python setup.py build && python setup.py install
-```
+or try to build it on your own (no advantages of doing that, except you want to change some dependency in the .def file):
 
-We'll also need some other tools and libraries
 ```bash
-conda install -c conda-forge matplotlib pyglet imageio
-pip install pygifsicle
-pip install openTSNE
+apptainer build retinal-rl.sif resources/retinal-rl.def
 ```
-IPython might also be necessary:
+
+### Prepare config directory for experiments
+
+The repository comes with some example configuration files, which you find under 'resources/config_templates'. For running experiments however, they need to be in 'config'.
+You can either copy them there by hand or run the following script from the top-level directory:
+
 ```bash
-conda install -c conda-forge ipython
+bash tests/ci/copy_configs.sh
 ```
 
-### Docker
+### Test basic functionality
 
-The `Dockerfile` is a thin wrapper around the berenslab `Dockerfile` for the berenslab cluster, but may still serve as a basis for developing a `docker` container for other systems. Regardless, after building the image we then create the `conda` environment as above.
+Now you are basically ready to run experiments!
+To test that everything is working fine, you can run:
 
-## Running retinal RL simulations
+```bash
+bash tests/ci/scan_configs.sh
+```
+
+The script loops over all experiments defined in config/experiment and runs a "scan" on them.
+If instead you want to run a single experiment file, run:
 
-Now that we have a (hopefully) working environment, we clone the repo
 ```bash
-https://github.com/berenslab/retinal-rl.git
+apptainer exec retinal-rl.sif python main.py +experiment="$experiment" command=scan system.device=cpu
 ```
+
+## Running retinal RL simulations [DEPRECATED]
+
 There are three main scripts for working with `retinal-rl`:
 
 - `train.py`: Train a model.

tests/ci/copy_configs.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+#===============================================================================
+# Description: Copies the config files from the resources dir to the config dir.
+#              Thus creates the basic structure needed to setup or run
+#              experiments.
+#
+# Usage:
+#   tests/ci/copy_configs.sh
+#   (run from top level directory!)
+#===============================================================================
+
+cp -r resources/config_templates/* config/

tests/ci/pylint.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+#===============================================================================
+# Description: Runs pylint either on all Python files or only on changed files
+#             compared to master branch using a specified Singularity container
+#
+# Arguments:
+#   $1 - Path to Singularity (.sif) container
+#   $2 - Optional: "--all" to run on all files, otherwise runs only on changed files
+#
+# Usage:
+#   tests/ci/run_pylint.sh container.sif         # Lint only changed Python files
+#   tests/ci/run_pylint.sh container.sif --all   # Lint all Python files
+#   (run from top level directory!)
+#
+# Dependencies:
+#   - Singularity/Apptainer + container
+#   - Container must have pylint installed
+#===============================================================================
+
+if [ "$2" = "--all" ]; then
+    apptainer exec "$1" pylint .
+else
+    changed_files=$(git diff --name-only origin/master...HEAD -- '*.py')
+    if [ -n "$changed_files" ]; then
+        apptainer exec "$1" pylint $(git diff --name-only origin/master...HEAD -- '*.py')
+    else
+        echo "No .py files changed"
+    fi
+fi