release: merge dev with main for release 3.0.0

.github/actions/install-python-and-package/action.yml

@@ -27,8 +27,10 @@ runs:
       with:
         update-conda: true
         python-version: ${{ inputs.python-version }}
-        conda-channels: anaconda
-    - run: conda --version
+        conda-channels: pytorch, pyg, bioconda, defaults, sbl, conda-forge
+    - run: |
+        conda --version
+        conda env list
       shell: bash {0}
     - name: Python info
       shell: bash -e {0}
@@ -41,16 +43,16 @@ runs:
         CMAKE_INSTALL_PREFIX: .local
       if: runner.os == 'Linux'
       run: |
-        # Install dependencies not handled by setuptools
+        # Install deeprank2 conda dependencies
         ## DSSP
-        sudo apt-get install -y dssp
+        conda install -c sbl dssp>=4.2.2.1
         ## MSMS
-        conda install -c bioconda msms
+        conda install -c bioconda msms>=2.6.1
         ## PyTorch, PyG, PyG adds
         ### Installing for CPU only on the CI
-        conda install pytorch==2.0.1 torchvision==0.15.2 torchaudio==2.0.2 -c pytorch
-        pip install torch_geometric==2.3.1
-        pip install torch_scatter torch_sparse torch_cluster torch_spline_conv -f https://data.pyg.org/whl/torch-$(python3 -c "import torch; print(torch.__version__)")+cpu.html
+        conda install pytorch=2.1.1 torchvision=0.16.1 torchaudio=2.1.1 cpuonly=2.0.* -c pytorch
+        conda install pyg=2.4.0 -c pyg
+        pip install torch_scatter==2.1.2 torch_sparse==0.6.18 torch_cluster==1.6.3 torch_spline_conv==1.2.2 -f https://data.pyg.org/whl/torch-2.1.0+cpu.html
     - name: Install dependencies on MacOS
       shell: bash {0}
       env:
@@ -59,26 +61,7 @@ runs:
       run: |
         # Install dependencies not handled by setuptools
         ## DSSP
-        git clone https://github.com/PDB-REDO/libcifpp.git --recurse-submodules
-        cd libcifpp
-        cmake -S . -B build -DCMAKE_INSTALL_PREFIX=$HOME/.local -DCMAKE_BUILD_TYPE=Release
-        cmake --build build
-        cmake --install build
-        #######
-        git clone https://github.com/mhekkel/libmcfp.git
-        cd libmcfp
-        mkdir build
-        cd build
-        cmake ..
-        cmake --build .
-        cmake --install .
-        #######
-        git clone https://github.com/PDB-REDO/dssp.git
-        cd dssp
-        mkdir build
-        cmake -S . -B build -DCMAKE_BUILD_TYPE=Release
-        cmake --build build
-        cmake --install build
+        conda install -c sbl dssp>=4.2.2.1
         ## MSMS
         cd /tmp/
         wget http://mgltools.scripps.edu/downloads/tars/releases/MSMSRELEASE/REL2.6.1/msms_i86Linux2_2.6.1.tar.gz

.github/workflows/build.yml

@@ -37,7 +37,7 @@ jobs:
       fail-fast: false
       matrix:
         os: ["ubuntu-latest"]
-        python-version: ["3.10", "3.11"]
+        python-version: ["3.10"] # ["3.10", "3.11"]
 
     steps:
       - uses: actions/checkout@v3

.github/workflows/coveralls.yml

@@ -46,7 +46,7 @@ jobs:
           python-version: ${{ matrix.python-version }}
           extras-require: test
       - name: Run unit tests with coverage
-        run: pytest --cov --cov-append --cov-report xml --cov-report term --cov-report html
+        run: pytest --cov --cov-append --cov-report xml --cov-fail-under=80 --cov-report term --cov-report html
       - name: Coveralls
         env:
           GITHUB_TOKEN: ${{ secrets.github_token }}

Dockerfile

@@ -2,31 +2,27 @@
 FROM --platform=linux/x86_64 condaforge/miniforge3:23.3.1-1
 
 # Add files
-ADD ./tutorials /home/deeprank2/tutorials 
+ADD ./tutorials /home/deeprank2/tutorials
 ADD ./env/environment.yml /home/deeprank2
 ADD ./env/requirements.txt /home/deeprank2
 
 # Install
 RUN \
-  apt update -y && \
-  apt install unzip -y && \
+  apt update -y &&
+  apt install unzip -y &&
   ## GCC
-  apt install -y gcc && \
-  ## DSSP
-  wget https://github.com/PDB-REDO/dssp/releases/download/v4.4.0/mkdssp-4.4.0-linux-x64 && \
-  mv mkdssp-4.4.0-linux-x64 /usr/local/bin/mkdssp && \
-  chmod a+x /usr/local/bin/mkdssp && \
+  apt install -y gcc &&
   ## Conda and pip deps
-  mamba env create -f /home/deeprank2/environment.yml && \
+  mamba env create -f /home/deeprank2/environment.yml &&
   ## Get the data for running the tutorials
-  if [ -d "/home/deeprank2/tutorials/data_raw" ]; then rm -Rf /home/deeprank2/tutorials/data_raw; fi && \
-  if [ -d "/home/deeprank2/tutorials/data_processed" ]; then rm -Rf /home/deeprank2/tutorials/data_processed; fi && \
-  wget https://zenodo.org/records/8349335/files/data_raw.zip && \
-  unzip data_raw.zip -d data_raw && \
+  if [ -d "/home/deeprank2/tutorials/data_raw" ]; then rm -Rf /home/deeprank2/tutorials/data_raw; fi &&
+  if [ -d "/home/deeprank2/tutorials/data_processed" ]; then rm -Rf /home/deeprank2/tutorials/data_processed; fi &&
+  wget https://zenodo.org/records/8349335/files/data_raw.zip &&
+  unzip data_raw.zip -d data_raw &&
   mv data_raw /home/deeprank2/tutorials
 
 # Activate the environment
-RUN echo "source activate deeprank2" > ~/.bashrc
+RUN echo "source activate deeprank2" >~/.bashrc
 ENV PATH /opt/conda/envs/deeprank2/bin:$PATH
 
 # Define working directory

docs/features.md

@@ -22,6 +22,59 @@ def add_features(
     pass
 ```
 
+Additionally, the nomenclature of the custom feature should be added in `deeprank2.domain.edgestorage` or `deeprank2.domain.nodestorage`, depending on which type of feature it is.
+
+As an example, this is the implementation of the node feature `res_type`, which represents the one-hot encoding of the amino acid residue and is defined in `deeprank2.features.components` module:
+
+```python
+from deeprank2.domain import nodestorage as Nfeat
+from deeprank2.molstruct.atom import Atom
+from deeprank2.molstruct.residue import Residue, SingleResidueVariant
+from deeprank2.utils.graph import Graph
+
+def add_features(
+    pdb_path: str, graph: Graph,
+    single_amino_acid_variant: Optional[SingleResidueVariant] = None
+    ):
+
+    for node in graph.nodes:
+        if isinstance(node.id, Residue):
+            residue = node.id
+        elif isinstance(node.id, Atom):
+            atom = node.id
+            residue = atom.residue
+        else:
+            raise TypeError(f"Unexpected node type: {type(node.id)}")
+
+        node.features[Nfeat.RESTYPE] = residue.amino_acid.onehot
+```
+
+`RESTYPE` is the name of the variable assigned to the feature `res_type` in `deeprank2.domain.nodestorage`. In order to use the feature from DeepRank2 API, its module needs to be imported and specified during the queries processing:
+
+```python
+from deeprank2.features import components
+
+feature_modules = [components]
+
+# Save data into 3D-graphs only
+hdf5_paths = queries.process(
+    "<output_folder>/<prefix_for_outputs>",
+    feature_modules = feature_modules)
+```
+
+Then, the feature `res_type` can be used from the DeepRank2 datasets API:
+
+```python
+from deeprank2.dataset import GraphDataset
+
+node_features = ["res_type"]
+
+dataset = GraphDataset(
+    hdf5_path = hdf5_paths,
+    node_features = node_features
+)
+```
+
 The following is a brief description of the features already implemented in the code-base, for each features' module.
 
 ## Default node features

docs/installation.md

@@ -1,53 +1,62 @@
-# Installations
+# Table of contents
 
-The package officially supports ubuntu-latest OS only, whose functioning is widely tested through the continuous integration workflows.
+- [Table of contents](#table-of-contents)
+- [Installation](#installation)
+  - [Containerized Installation](#containerized-installation)
+  - [Local/remote installation](#localremote-installation)
+      - [YML file installation](#yml-file-installation)
+    - [Manual installation](#manual-installation)
+    - [Testing DeepRank2 installation](#testing-deeprank2-installation)
+- [Contributing](#contributing)
 
-You can either install DeepRank2 in a [dockerized container](#containerized-installation), which will allow you to run our [tutorial notebooks](https://github.com/DeepRank/deeprank2/tree/main/tutorials), or you can [install the package locally](#localremote-installation).
+# Installation
 
-## Containerized Installation 
+There are two ways to install DeepRank2:
 
-In order to try out the package without worrying about your OS and without the need of installing all the required dependencies, we created a `Dockerfile` that can be used for taking care of everything in a suitable container. After having cloned the repository and installed [Docker](https://docs.docker.com/engine/install/), run the following commands (you may need to have sudo permission) from the root of the repository.
+1. In a [dockerized container](#containerized-installation). This allows you to use DeepRank2, including all the notebooks within the container (a protected virtual space), without worrying about your operating system or installation of dependencies.
+   - We recommend this installation for inexperienced users and to learn to use or test our software, e.g. using the provided [tutorials](tutorials/TUTORIAL.md). However, resources might be limited in this installation and we would not recommend using it for large datasets or on high-performance computing facilities.
+2. [Local installation](#localremote-installation) on your system. This allows you to use the full potential of DeepRank2, but requires a few additional steps during installation.
+   - We recommend this installation for more experienced users, for larger projects, and for (potential) [contributors](#contributing) to the codebase.
 
-Build the Docker image:
+## Containerized Installation
 
-```bash
-docker build -t deeprank2 .
-```
+In order to try out the package without worrying about your OS and without the need of installing all the required dependencies, we created a `Dockerfile` that can be used for taking care of everything in a suitable container.
 
-Run the Docker container:
+For this, you first need to install [Docker](https://docs.docker.com/engine/install/) on your system. Then run the following commands. You may need to have sudo permission for some steps, in which case the commands below can be preceded by `sudo`:
 
 ```bash
+# Clone the DeepRank2 repository and enter its root directory
+git clone https://github.com/DeepRank/deeprank2
+cd deeprank2
+
+# Build and run the Docker image
+docker build -t deeprank2 .
 docker run -p 8888:8888 deeprank2
 ```
 
-This assumes that your application inside the container is listening on port 8888, and you want to map it to port 8888 on your host machine. Open a browser and go to `http://localhost:8888` to access the application running inside the Docker container and run the tutorials' notebooks.
+Next, open a browser and go to `http://localhost:8888` to access the application running inside the Docker container. From there you can use DeepRank2, e.g. to run the tutorial notebooks.
 
-More details about the tutorials' content can be found [here](https://github.com/DeepRank/deeprank2/blob/main/tutorials/TUTORIAL.md). Note that in the docker container only the raw PDB files are downloaded, needed as a starting point for the tutorials. You can obtain the processed HDF5 files by running the `data_generation_xxx.ipynb` notebooks. Because Docker containers are limited in memory resources, we limit the number of data points processed in the tutorials'. Please install the package locally to fully leverage its capabilities.
+More details about the tutorials' contents can be found [here](https://github.com/DeepRank/deeprank2/blob/main/tutorials/TUTORIAL.md). Note that in the docker container only the raw PDB files are downloaded, which needed as a starting point for the tutorials. You can obtain the processed HDF5 files by running the `data_generation_xxx.ipynb` notebooks. Because Docker containers are limited in memory resources, we limit the number of data points processed in the tutorials. Please [install the package locally](#localremote-installation) to fully leverage its capabilities.
 
-After running the tutorials, you may want to remove the (quite large) Docker image from your machine. In this case, remember to [stop the container](https://docs.docker.com/engine/reference/commandline/stop/) and then [remove the image](https://docs.docker.com/engine/reference/commandline/image_rm/). More general information about Docker can be found on the [official website docs](https://docs.docker.com/get-started/).
+If after running the tutorials you want to remove the (quite large) Docker image from your machine, you must first [stop the container](https://docs.docker.com/engine/reference/commandline/stop/) and can then [remove the image](https://docs.docker.com/engine/reference/commandline/image_rm/). More general information about Docker can be found on the [official website docs](https://docs.docker.com/get-started/).
 
 ## Local/remote installation
 
-### Non-pythonic dependencies
-
-Instructions are up to date as of 27 Nov 2023.
-
-Before installing deeprank2 you need to install some dependencies:
-
-*  [DSSP 4](https://swift.cmbi.umcn.nl/gv/dssp/)
-    * Check if `dssp` is installed: `dssp --version`. If this gives an error or shows a version lower than 4:
-      * on ubuntu 22.04 or newer: `sudo apt-get install dssp`. If the package cannot be located, first run `sudo apt-get update`.
-      * on older versions of ubuntu or on mac or lacking sudo priviliges: install from [here](https://github.com/pdb-redo/dssp), following the instructions listed. Alternatively, follow [this](https://github.com/PDB-REDO/libcifpp/issues/49) thread. 
-*  [GCC](https://gcc.gnu.org/install/)
-    * Check if gcc is installed: `gcc --version`. If this gives an error, run `sudo apt-get install gcc`. 
+Local installation is formally only supported on the latest stable release of ubuntu, for which widespread automated testing through continuous integration workflows has been set up. However, it is likely that the package runs smoothly on other operating systems as well.
 
-### Pythonic dependencies
+Before installing DeepRank2 please ensure you have [GCC](https://gcc.gnu.org/install/) installed: if running `gcc --version` gives an error, run `sudo apt-get install gcc`.
 
-Instructions are up to date as of 27 Nov 2023.
+#### YML file installation
 
-Then, you can use the YML file we provide for creating a [conda environment](https://conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html) containing the latest stable release of the package and all the other necessary conda and pip dependencies (CPU only, Python 3.10):
+You can use the provided YML file for creating a [conda environment](https://conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html) containing the latest stable release of DeepRank2 and all its dependencies.
+This will install the CPU-only version of DeepRank2 on Python 3.10.
+Note that this will not work for MacOS. Do the [Manual Installation](#manual-installation) instead.
 
 ```bash
+# Clone the DeepRank2 repository and enter its root directory
+git clone https://github.com/DeepRank/deeprank2
+cd deeprank2
+
 # Ensure you are in your base environment
 conda activate
 # Create the environment
@@ -56,26 +65,24 @@ conda env create -f env/environment.yml
 conda activate deeprank2
 ```
 
-Alternatively, if you are a MacOS user, if the YML file installation is not successfull, or if you want to use CUDA or Python 3.11, you can install each dependency separately, and then the latest stable release of the package using the PyPi package manager. Also in this case, we advise to use a [conda environment](https://conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html). In case of issues during installation, please refer to the official documentation for each package (linked below), as our instructions may be out of date:
+See instructions below to [test](#testing-deeprank2-installation) that the installation was succesful.
+
+### Manual installation
+
+If you want to use the GPUs, choose a specific python version, are a MacOS user, or if the YML installation was not succesful, you can install the package manually. We advise to do this inside a [conda virtual environment](https://conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html).
+If you have any issues during installation of dependencies, please refer to the official documentation for each package (linked below), as our instructions may be out of date (last tested on 19 Jan 2024):
 
-- [MSMS](https://anaconda.org/bioconda/msms): `conda install -c bioconda msms`.
+- [DSSP 4](https://anaconda.org/sbl/dssp): `conda install -c sbl dssp`
+- [MSMS](https://anaconda.org/bioconda/msms): `conda install -c bioconda msms`
   - [Here](https://ssbio.readthedocs.io/en/latest/instructions/msms.html) for MacOS with M1 chip users.
-- [PyTorch](https://pytorch.org/get-started/locally/)
+- [PyTorch](https://pytorch.org/get-started/locally/): `conda install pytorch torchvision torchaudio cpuonly -c pytorch`
+  - Pytorch regularly publishes updates and not all newest versions will work stably with DeepRank2. Currently, the package is tested using [PyTorch 2.1.1](https://pytorch.org/get-started/previous-versions/#v211).
   - We support torch's CPU library as well as CUDA.
 - [PyG](https://pytorch-geometric.readthedocs.io/en/latest/install/installation.html) and its optional dependencies: `torch_scatter`, `torch_sparse`, `torch_cluster`, `torch_spline_conv`.
-- [DSSP 4](https://swift.cmbi.umcn.nl/gv/dssp/)
-- Check if `dssp` is installed: `dssp --version`. If this gives an error or shows a version lower than 4:
-- on ubuntu 22.04 or newer: `sudo apt-get install dssp`. If the package cannot be located, first run `sudo apt-get update`.
-- on older versions of ubuntu or on mac or lacking sudo priviliges: install from [here](https://github.com/pdb-redo/dssp), following the instructions listed.
-- [GCC](https://gcc.gnu.org/install/)
-- Check if gcc is installed: `gcc --version`. If this gives an error, run `sudo apt-get install gcc`.
-- For MacOS with M1 chip users only install [the conda version of PyTables](https://www.pytables.org/usersguide/installation.html).
+  - The exact command to install pyg will depend on the version of pytorch you are using. Please refer to the source's installation instructions (we recommend using the pip installation for this as it also shows the command for the dependencies).
+- For MacOS with M1 chip users: install [the conda version of PyTables](https://www.pytables.org/usersguide/installation.html).
 
-Finally do:
-
-```bash
-pip install deeprank2
-```
+Finally install deeprank2 itself: `pip install deeprank2`.
 
 Alternatively, get the latest updates by cloning the repo and installing the editable version of the package with:
 
@@ -85,18 +92,16 @@ cd deeprank2
 pip install -e .'[test]'
 ```
 
-The `test` extra is optional, and can be used to install test-related dependencies useful during the development.
+The `test` extra is optional, and can be used to install test-related dependencies, useful during development.
 
-### Test installation
+### Testing DeepRank2 installation
+
+You can check that all components were installed correctly, using pytest. We especially recommend doing this in case you installed DeepRank2 and its dependencies manually (the latter option above).
 
-If you have installed the package from a cloned repository (the latter option above), you can check that all components were installed correctly, using pytest (run `pip install pytest` if you did not install it above).
 The quick test should be sufficient to ensure that the software works, while the full test (a few minutes) will cover a much broader range of settings to ensure everything is correct.
 
-Run `pytest tests/test_integration.py` for the quick test or just `pytest` for the full test (expect a few minutes to run).
+First run `pip install pytest`, if you did not install it above. Then run `pytest tests/test_integration.py` for the quick test or just `pytest` for the full test (expect a few minutes to run).
 
 # Contributing
 
 If you would like to contribute to the package in any way, please see [our guidelines](CONTRIBUTING.rst).
-
-The following section serves as a first guide to start using the package, using protein-protein Interface (PPI) queries as example. For an enhanced learning experience, we provide in-depth [tutorial notebooks](https://github.com/DeepRank/deeprank2/tree/main/tutorials) for generating PPI data, generating SVR data, and for the training pipeline.
-For more details, see the [extended documentation](https://deeprank2.rtfd.io/).

env/environment.yml

@@ -4,10 +4,13 @@ channels:
   - pyg
   - bioconda
   - defaults
+  - conda-forge
+  - sbl
 dependencies:
   - pip==23.3.*
   - python==3.10.*
   - msms==2.6.1
+  - dssp>=4.2.2.1
   - pytorch==2.1.1
   - pytorch-mutex==1.0.*
   - torchvision==0.16.1
@@ -16,4 +19,4 @@ dependencies:
   - pyg==2.4.0
   - notebook==7.0.6
   - pip:
-    - --requirement requirements.txt
+      - --requirement requirements.txt

tutorials/data_generation_ppi.ipynb

@@ -145,7 +145,7 @@
     "pdb_files, bas = get_pdb_files_and_target_data(data_path)\n",
     "\n",
     "if limit_data:\n",
-    "\tpdb_files = pdb_files[:15]"
+    "    pdb_files = pdb_files[:15]"
    ]
   },
   {
@@ -213,7 +213,7 @@
     "    if count % 20 == 0:\n",
     "        print(f\"{count} queries added to the collection.\")\n",
     "\n",
-    "print(f\"Queries ready to be processed.\\n\")"
+    "print(\"Queries ready to be processed.\\n\")"
    ]
   },
   {
@@ -255,7 +255,9 @@
     "    grid_map_method=grid_map_method,\n",
     ")\n",
     "\n",
-    "print(f'The queries processing is done. The generated HDF5 files are in {os.path.join(processed_data_path, \"residue\")}.')"
+    "print(\n",
+    "    f'The queries processing is done. The generated HDF5 files are in {os.path.join(processed_data_path, \"residue\")}.'\n",
+    ")"
    ]
   },
   {
@@ -358,7 +360,9 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "fname = os.path.join(processed_data_path, \"residue\", \"_\".join([\"res_mass\", \"distance\", \"electrostatic\"]))\n",
+    "fname = os.path.join(\n",
+    "    processed_data_path, \"residue\", \"_\".join([\"res_mass\", \"distance\", \"electrostatic\"])\n",
+    ")\n",
     "dataset.save_hist(features=[\"res_mass\", \"distance\", \"electrostatic\"], fname=fname)\n",
     "\n",
     "im = img.imread(fname + \".png\")\n",
@@ -450,7 +454,7 @@
     "    if count % 20 == 0:\n",
     "        print(f\"{count} queries added to the collection.\")\n",
     "\n",
-    "print(f\"Queries ready to be processed.\\n\")"
+    "print(\"Queries ready to be processed.\\n\")"
    ]
   },
   {
@@ -476,7 +480,9 @@
     "    grid_map_method=grid_map_method,\n",
     ")\n",
     "\n",
-    "print(f'The queries processing is done. The generated HDF5 files are in {os.path.join(processed_data_path, \"atomic\")}.')"
+    "print(\n",
+    "    f'The queries processing is done. The generated HDF5 files are in {os.path.join(processed_data_path, \"atomic\")}.'\n",
+    ")"
    ]
   },
   {

tutorials/data_generation_srv.ipynb

@@ -151,7 +151,7 @@
     "pdb_files, res_numbers, res_wildtypes, res_variants, targets = get_pdb_files_and_target_data(data_path)\n",
     "\n",
     "if limit_data:\n",
-    "\tpdb_files = pdb_files[:15]"
+    "    pdb_files = pdb_files[:15]"
    ]
   },
   {
@@ -266,7 +266,9 @@
     "    grid_map_method=grid_map_method,\n",
     ")\n",
     "\n",
-    "print(f'The queries processing is done. The generated HDF5 files are in {os.path.join(processed_data_path, \"residue\")}.')"
+    "print(\n",
+    "    f'The queries processing is done. The generated HDF5 files are in {os.path.join(processed_data_path, \"residue\")}.'\n",
+    ")"
    ]
   },
   {
@@ -376,7 +378,9 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "fname = os.path.join(processed_data_path, \"residue\", \"_\".join([\"res_mass\", \"distance\", \"electrostatic\"]))\n",
+    "fname = os.path.join(\n",
+    "    processed_data_path, \"residue\", \"_\".join([\"res_mass\", \"distance\", \"electrostatic\"])\n",
+    ")\n",
     "dataset.save_hist(features=[\"res_mass\", \"distance\", \"electrostatic\"], fname=fname)\n",
     "\n",
     "im = img.imread(fname + \".png\")\n",
@@ -470,7 +474,7 @@
     "    if count % 20 == 0:\n",
     "        print(f\"{count} queries added to the collection.\")\n",
     "\n",
-    "print(f\"Queries ready to be processed.\\n\")"
+    "print(\"Queries ready to be processed.\\n\")"
    ]
   },
   {
@@ -496,7 +500,9 @@
     "    grid_map_method=grid_map_method,\n",
     ")\n",
     "\n",
-    "print(f'The queries processing is done. The generated HDF5 files are in {os.path.join(processed_data_path, \"atomic\")}.')"
+    "print(\n",
+    "    f'The queries processing is done. The generated HDF5 files are in {os.path.join(processed_data_path, \"atomic\")}.'\n",
+    ")"
    ]
   },
   {