upgrade tox test; create lint workflow (#349)

* update test workflow settings

* upgrade install script, use py39

* add pandas version clause for py39

* add notest option to tox

* upload lint workflow config

* update cuda path, fix python version to base python, no need to use tox-go-actions

* fix: specify runs-on

* fix: pre-commit py38 clause; drop py312; fix RUN_SETTINGS names

* fix python_version clause; fix py36 in ubuntu

* fix: change order to prevent early overwritting

* clean up unused

* fix: tox py38 clause

* fix: gha python version dict; pyg-lib

* drop py36

* upgrade numpy and numba

* downgrade numpy for compatibilty with numba

* add py37 and py311 support in setup cfg

* drop py37

* disable pyg-lib due to incompatibillity with windows

* readme: installation versions

* min require py38

.github/dependabot.yml

@@ -15,4 +15,3 @@ updates:
     ignore:
       - dependency-name: "torch" # see PyG
       - dependency-name: "torchvision"
-      - dependency-name: "numpy" # see numba

.github/workflows/lint.yml

@@ -0,0 +1,34 @@
+name: Lint
+
+on:
+  push:
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  run_lint_and_install_test:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest]
+        python-version: ['3.8', '3.9', '3.10', '3.11']
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          python -m pip install -U pip
+          pip install tox tox-gh-actions -U
+
+      - name: Lint and test installation with tox
+        run: tox
+        env:
+          RUN_SETTINGS: cpu-notest

.github/workflows/test_examples.yml

@@ -6,17 +6,18 @@ on:
     - cron: '59 23 * * 0,2,4'
 
 env:
-  # CUDA paths for MSU ICER HPC with "module load GCC/8.3.0 CUDA/10.2.89"
-  CUDA_PATH: '/opt/software/CUDA/10.2.89-GCC-8.3.0'
-  LD_LIBRARY_PATH: '/opt/software/CUDA/10.2.89-GCC-8.3.0/lib64'
+  ### CUDA paths for MSU ICER HPC with "module load CUDA/11.8"
+  CUDA_PATH: '/opt/software/CUDA/11.8.0'
+  LD_LIBRARY_PATH: '/opt/software/CUDA/11.8.0/lib64'
+  ### CUDA paths for MSU ICER HPC with "module load GCC/8.3.0 CUDA/10.2.89"
+  # CUDA_PATH: '/opt/software/CUDA/10.2.89-GCC-8.3.0'
+  # LD_LIBRARY_PATH: '/opt/software/CUDA/10.2.89-GCC-8.3.0/lib64'
 
 jobs:
   run_examples:
     runs-on: self-hosted
     strategy:
       fail-fast: false
-      matrix:
-        python-version: ['3.8']
 
     steps:
       - uses: actions/checkout@v3
@@ -27,7 +28,7 @@ jobs:
       - name: Install dependencies
         run:
           python -m pip install -U pip
-          pip install tox tox-gh-actions -U
+          pip install tox -U
 
       - name: Test with tox
-        run: tox -e python${{ matrix.python-version }}-gpu
+        run: tox -e py38-gpu-test

README.md

@@ -110,8 +110,8 @@ as well as easily reproducible experiments by providing unified tools for
 The full installation process might be a bit tedious and could involve some debugging when using CUDA enabled packages.
 Thus, we provide an `install.sh` script that simplifies the installation process, assuming the user have [conda](https://conda.io/projects/conda/en/latest/index.html) set up on their machines.
 The installation script creates a conda environment `dance` and install the DANCE package along with all its dependencies with a apseicifc CUDA version.
-Currently, two options are accepted: `cpu` and  `cu117`.
-For example, to install the DANCE package using CUDA11.7 in a `dance-env` conda environment, simply run:
+Currently, two options are accepted: `cpu` and  `cu118`.
+For example, to install the DANCE package using CUDA 11.8 in a `dance-env` conda environment, simply run:
 
 ```bash
 # Clone the repository via SSH
@@ -120,7 +120,7 @@ git clone [email protected]:OmicsML/dance.git && cd dance
 # git clone https://github.com/OmicsML/dance.git  && cd dance
 
 # Run the auto installation script to install DANCE and its dependencies in a conda environment
-source install.sh cu117 dance-env
+source install.sh cu118 dance-env
 ```
 
 **Note**: the first argument for cuda version is mandatory, while the second argument for conda environment name is optional (default is `dance`).
@@ -140,17 +140,17 @@ conda create -n dance python=3.8 -y && conda activate dance-dev
 Then, install CUDA enabled packages (PyTorch, PyG, DGL):
 
 ```bash
-conda install pytorch=2.0.0 torchvision torchaudio pytorch-cuda=11.7 -c pytorch -c nvidia -y
-conda install pyg=2.3.0 -c pyg -y
-conda install dgl=1.0.1 -c dglteam/label/cu117 -y
+conda install pytorch=2.0.1 torchvision torchaudio pytorch-cuda=11.8 -c pytorch -c nvidia -y
+conda install pyg=2.3.1 -c pyg -y
+conda install dgl=1.1.2 -c dglteam/label/cu118 -y
 ```
 
 Alternatively, install these dependencies for CPU only:
 
 ```bash
-conda install pytorch=2.0.0 torchvision torchaudio cpuonly -c pytorch -y
-conda install pyg=2.3.0 -c pyg -y
-conda install dgl -c dglteam -y
+conda install pytorch=2.0.1 torchvision torchaudio cpuonly -c pytorch -y
+conda install pyg=2.3.1 -c pyg -y
+conda install dgl=1.1.2 -c dglteam -y
 ```
 
 For more information about installation or other CUDA version options, check out the installation pages for the corresponding packages

install.sh

@@ -4,7 +4,7 @@
 # as PyTorch, Pytorch Geometric (PYG), and Deep Graph Library (DGL).
 #
 # Example:
-# $ source install.sh cu117  # install dance with CUDA 11.7
+# $ source install.sh cu118  # install dance with CUDA 11.8
 #
 # To uninstall and remove the dance environment:
 # $ conda remove -n dance --all
@@ -13,7 +13,7 @@ trap "echo Try using source instead of sh? && trap - ERR && return 1" ERR
 
 # Check required version specification input
 if [ -z $1 ]; then
-    echo "ERROR: Please provide CUDA information, available options are [cpu,cu117]"
+    echo "ERROR: Please provide CUDA information, available options are [cpu,cu118]"
     return 1
 fi
 
@@ -27,11 +27,11 @@ else
 fi
 
 # Torch related dependency versions
-PYTORCH_VERSION=2.0.0
-TORCHVISION_VERSION=0.15.0
-TORCHAUDIO_VERSION=2.0.0
-PYG_VERSION=2.3.0
-DGL_VERSION=1.0.1
+PYTORCH_VERSION=2.0.1
+TORCHVISION_VERSION=0.15.2
+TORCHAUDIO_VERSION=2.0.2
+PYG_VERSION=2.3.1
+DGL_VERSION=1.1.2
 
 # Set CUDA variable (use CPU if not set)
 CUDA_VERSION=${1:-cpu}
@@ -41,9 +41,9 @@ case $CUDA_VERSION in
         PYTORCH_CUDA_OPT="cpuonly -c pytorch"
         DGL_CHANNEL="dglteam"
         ;;
-    cu117)
-        PYTORCH_CUDA_OPT="pytorch-cuda=11.7 -c pytorch -c nvidia"
-        DGL_CHANNEL="dglteam/label/cu117"
+    cu118)
+        PYTORCH_CUDA_OPT="pytorch-cuda=11.8 -c pytorch -c nvidia"
+        DGL_CHANNEL="dglteam/label/cu118"
         ;;
     *)
         echo "ERROR: Unrecognized CUDA_VERSION=${CUDA_VERSION}"
@@ -52,7 +52,7 @@ case $CUDA_VERSION in
 esac
 
 # Create environment
-conda create -n ${envname} python=3.8 -y
+conda create -n ${envname} python=3.9 -y
 conda activate ${envname}
 
 # Install CUDA enabled dependencies

requirements.txt

@@ -3,11 +3,13 @@ igraph==0.10.8
 leidenalg==0.10.1
 mudata==0.2.3
 networkx==3.1
-numba==0.57.1
-numpy==1.22.4
+numba==0.58.0
+numpy==1.24.4; python_version < "3.9"
+numpy==1.25.2; python_version >= "3.9"
 opencv-python==4.8.0.76
 openpyxl==3.1.2
-pandas==2.1.1
+pandas==2.0.3; python_version < "3.9"
+pandas==2.1.1; python_version >= "3.9"
 pyro-ppl==1.8.6
 requests==2.31.0
 scanpy==1.9.4

setup.cfg

@@ -23,6 +23,7 @@ classifiers =
     Programming Language :: Python :: 3.8
     Programming Language :: Python :: 3.9
     Programming Language :: Python :: 3.10
+    Programming Language :: Python :: 3.11
     Programming Language :: Python :: 3 :: Only
 keywords =
     Single-cell Biology
@@ -60,11 +61,13 @@ packages = find:
 
 [options.extras_require]
 dev =
-    pre-commit==3.4.0
+    pre-commit==2.21.0; python_version < "3.8"
+    pre-commit==3.4.0; python_version >= "3.8"
     pytest-subtests==0.11.0
     pytest-xdist==3.3.1
     pytest==7.4.2
-    tox==4.11.3
+    tox==4.8.0; python_version < "3.8"
+    tox==4.11.3; python_version >= "3.8"
 doc =
     sphinx
     sphinx-autodoc-typehints

tox.ini

@@ -1,30 +1,66 @@
 [tox]
 minversion = 3.8.0
-envlist = python3.8-{osx,cpu,gpu}
+envlist = py3{7,8,9,10,11}-{cpu,osx,gpu}-{notest,test}
 isolated_build = true
 
+[gh-actions]
+python =
+    3.8: py38
+    3.9: py39
+    3.10: py310
+    3.11: py311
+
+[gh-actions:env]
+RUN_SETTINGS =
+    cpu-test: cpu-test
+    osx-test: osx-test
+    gpu-test: gpu-test
+    cpu-notest: cpu-notest
+    osx-notest: osx-notest
+    gpu-notest: gpu-notest
+
 [testenv]
 setenv =
     PYTHONPATH = {toxinidir}
 extras =
     dev
 commands =
-    # First upgrade pip
     pip install -U pip
 
-    # Install torch related dependencies with CUDA 10.2
-    gpu: pip install torch==1.12.1+cu102 torchvision==0.13.1+cu102 --extra-index-url https://download.pytorch.org/whl/cu102
-    cpu: pip install torch==1.12.1+cpu torchvision==0.13.1+cpu --extra-index-url https://download.pytorch.org/whl/cpu
-    osx: pip install torch==1.12.1 torchvision==0.13.1
+    # Install torch related dependencies with CUDA 11.8
+    gpu: pip install torch==2.0.1 torchvision==0.15.2 --index-url https://download.pytorch.org/whl/cu118
+    cpu: pip install torch==2.0.1 torchvision==0.15.2 --index-url https://download.pytorch.org/whl/cpu
+    osx: pip install torch==2.0.1 torchvision==0.15.2
+
     # Custom installation for PyG, cf. https://github.com/rusty1s/pytorch_scatter/pull/268
-    gpu: pip install torch-scatter torch-sparse torch-cluster torch-geometric -f https://data.pyg.org/whl/torch-1.12.1+cu102.html
-    {cpu,osx}: pip install torch-geometric==2.1.0 torch-scatter torch-sparse torch-cluster -f https://data.pyg.org/whl/torch-1.12.1+cpu.html
+    pip install torch_geometric==2.3.1
+    # gpu: pip install pyg-lib==0.2.0 -f https://data.pyg.org/whl/torch-2.0.0+cu118.html
+    # {cpu,osx}: pip install pyg-lib==0.2.0 -f https://data.pyg.org/whl/torch-2.0.0+cpu.html
+
     # Custom installation for DGL
-    gpu: pip install dgl-cu102 -f https://data.dgl.ai/wheels/repo.html
-    {cpu,osx}: pip install dgl -f https://data.dgl.ai/wheels/repo.html
+    gpu: pip install dgl==1.1.2 -f https://data.dgl.ai/wheels/cu118/repo.html
+    {cpu,osx}: pip install dgl==1.1.2 -f https://data.dgl.ai/wheels/repo.html
+
+    ### PyTorch 1.12.1, PyG 2.1.0, DGL 1.0, CUDA 10.2
+    # gpu: pip install torch==1.12.1+cu102 torchvision==0.13.1+cu102 --extra-index-url https://download.pytorch.org/whl/cu102
+    # cpu: pip install torch==1.12.1+cpu torchvision==0.13.1+cpu --extra-index-url https://download.pytorch.org/whl/cpu
+    # osx: pip install torch==1.12.1 torchvision==0.13.1
+    # # Custom installation for PyG, cf. https://github.com/rusty1s/pytorch_scatter/pull/268
+    # gpu: pip install torch-scatter torch-sparse torch-cluster torch-geometric -f https://data.pyg.org/whl/torch-1.12.1+cu102.html
+    # {cpu,osx}: pip install torch-geometric==2.1.0 torch-scatter torch-sparse torch-cluster -f https://data.pyg.org/whl/torch-1.12.1+cpu.html
+    # # Custom installation for DGL
+    # gpu: pip install dgl-cu102 -f https://data.dgl.ai/wheels/repo.html
+    # {cpu,osx}: pip install dgl -f https://data.dgl.ai/wheels/repo.html
 
     # Install the rest of the dependencies
     pip install -r {toxinidir}/requirements.txt
 
+    # Check installation
+    python --version
+    python -c "import torch; print(f'Successfully installed pytorch {torch.__version__}')"
+    python -c "import torch_geometric, torch_geometric.nn; print(f'Successfully installed pyg {torch_geometric.__version__}')"
+    python -c "import dgl; print(f'Successfully installed dgl {dgl.__version__}')"
+    python -c "import dance; print(f'Successfully installed dance {dance.__version__}')"
+
     # Run tests
-    pytest tests/ -v
+    test: pytest tests/ -v