Continuous integration

.github/workflows/build.yml

@@ -1,35 +1,146 @@
 name: nitorch-build
 
 on:
-  workflow_dispatch
-#  push:
-#    branches: [  ]
-#  pull_request:
-#    branches: [  ]
+  workflow_dispatch:
+    inputs:
+      fail-fast:
+        description: 'Fail fast mode'
+        required: false
+        default: 'true'
+  push:
+    paths:
+      # all c++/cuda source files
+      - "**.c"
+      - "**.cc"
+      - "**.cu"
+      - "**.cpp"
+      - "**.h"
+      - "**.cuh"
+      # github actions
+      - ".github/workflows/build.yml"
+      - "scripts/actions/**"
+      # all python files in the root directory
+      - "*.py"
+    branches: [ master ]
+  pull_request:
+    paths:
+      # all c++/cuda source files
+      - "**.c"
+      - "**.cc"
+      - "**.cu"
+      - "**.cpp"
+      - "**.h"
+      - "**.cuh"
+      # github actions
+      - ".github/workflows/build.yml"
+      - "scripts/actions/**"
+      # all python files in the root directory
+      - "*.py"
+    branches: [ master ]
 
 jobs:
 
   build:
 
-    runs-on: ${{ matrix.os }}
+    runs-on: ubuntu-latest
+#    runs-on: ${{ matrix.os }}
     strategy:
-      fail-fast: false
+      fail-fast: ${{ github.event.inputs.fail-fast == 'true' }}
       matrix:
-        os: [ubuntu-latest, macos-latest, windows-latest]
-        python-version: [ 3.6, 3.7, 3.8 ]
-        pytorch-version: [ 1.3, 1.4, 1.5, 1.6, 1.7 ]
+        os: [ ubuntu-latest ]
+        # os: [ ubuntu-latest, macos-latest, windows-latest ]
+        python-version: [ '3.6', '3.8' ]  # '3.7' fails for some weird reason
+        pytorch-version: [ '1.3', '1.4', '1.5', '1.6', '1.7' ]
+        cuda-version: [ 'cpu', '10.1', '10.2', '11.0' ]
+        exclude:
+        - pytorch-version: '1.3'
+          python-version: '3.8'
+        - pytorch-version: '1.3'
+          cuda-version: '10.2'
+        - pytorch-version: '1.4'
+          cuda-version: '10.2'
+        - pytorch-version: '1.3'
+          cuda-version: '11.0'
+        - pytorch-version: '1.4'
+          cuda-version: '11.0'
+        - pytorch-version: '1.5'
+          cuda-version: '11.0'
+        - pytorch-version: '1.6'
+          cuda-version: '11.0'
 
     steps:
+
     - uses: actions/checkout@v2
+
+    - if: matrix.cuda-version != 'cpu' && runner.os == 'Windows'
+      # Windows -> powershell
+      name: Install CUDA ${{ matrix.cuda-version }} (Windows)
+      env:
+        cuda: ${{ matrix.cuda-version }}
+      shell: powershell
+      run: |
+        # Install CUDA via a powershell script
+        .\scripts\actions\install_cuda_windows.ps1
+        if ($?) {
+          # Set paths for subsequent steps, using $env:CUDA_PATH
+          echo "Adding CUDA to CUDA_PATH, CUDA_PATH_X_Y and PATH"
+          echo "CUDA_PATH=$env:CUDA_PATH" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
+          echo "$env:CUDA_PATH_VX_Y=$env:CUDA_PATH" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
+          echo "$env:CUDA_PATH/bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
+        }
+
+    - if: matrix.cuda-version != 'cpu' && runner.os != 'Windows'
+      # Unix -> bash
+      name: Install CUDA ${{ matrix.cuda-version }} (Unix)
+      env:
+        cuda: ${{ matrix.cuda-version }}
+      shell: bash
+      run: |
+        if [ ! -z ${{ matrix.cuda-version }} ]; then
+          os="$(cut -d'-' -f1 <<< ${{ matrix.os }})"
+          echo "$os"
+          if [ ! -f "./scripts/actions/install_cuda_${os}.sh" ]; then
+            echo "cuda not available on ${os}"
+            exit 1
+          fi
+          source "./scripts/actions/install_cuda_${os}.sh"
+          if [[ $? -eq 0 ]]; then
+            # Set paths for subsequent steps, using ${CUDA_PATH}
+            echo "Adding CUDA to CUDA_PATH, PATH and LD_LIBRARY_PATH"
+            echo "CUDA_PATH=${CUDA_PATH}" >> $GITHUB_ENV
+            echo "${CUDA_PATH}/bin" >> $GITHUB_PATH
+            echo "LD_LIBRARY_PATH=${CUDA_PATH}/lib:${LD_LIBRARY_PATH}" >> $GITHUB_ENV
+          fi
+        fi
+
     - name: Set up Python ${{ matrix.python-version }} on ${{ matrix.os }}
       uses: actions/setup-python@v2
       with:
         python-version: ${{ matrix.python-version }}
+
     - name: Install pip
       run: |
         python -m pip install --upgrade pip
         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-    - name: Install PyTorch
-      run: pip install torch==${{ matrix.pytorch-version }}
-    - name: Build
-      run: python ./setup.py install
+
+    - name: Install PyTorch ${{ matrix.pytorch-version }}
+      env:
+        cuda: ${{ matrix.cuda-version }}
+        torch: ${{ matrix.pytorch-version }}
+      shell: bash
+      run: |
+        os="$(cut -d'-' -f1 <<< ${{ matrix.os }})"
+        echo "$os"
+        if [ ! -f "./scripts/actions/install_pytorch_${os}.sh" ]; then
+          echo "pytorch not available on ${os}"
+          exit 1
+        fi
+        source "./scripts/actions/install_pytorch_${os}.sh"
+
+    - name: Build nitorch
+      # Compiling for all architectures takes ages and this workflow is just
+      # here to test that we didn't break anything in the compilation chain
+      # so we only test one architecture (sm_35 -> the default)
+      # In the package distribution workflow, we'll need to compile for
+      # all architectures supported by (the pypi version of) pytorch.
+      run: TORCH_CUDA_ARCH_LIST="3.5" python setup.py install

.github/workflows/manual-build.yml

@@ -29,9 +29,29 @@ jobs:
     steps:
     - uses: actions/checkout@v2
 
-    - name: Install CUDA ${{ github.event.inputs.cuda-version }}
+    - if: runner.os == 'Windows'
+      # Windows -> powershell
+      name: Install CUDA ${{ github.event.inputs.cuda-version }} (Windows)
       env:
         cuda: ${{ github.event.inputs.cuda-version }}
+      shell: powershell
+      run: |
+        # Install CUDA via a powershell script
+        .\scripts\actions\install_cuda_windows.ps1
+        if ($?) {
+          # Set paths for subsequent steps, using $env:CUDA_PATH
+          echo "Adding CUDA to CUDA_PATH, CUDA_PATH_X_Y and PATH"
+          echo "CUDA_PATH=$env:CUDA_PATH" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
+          echo "$env:CUDA_PATH_VX_Y=$env:CUDA_PATH" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
+          echo "$env:CUDA_PATH/bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
+        }
+
+    - if: runner.os != 'Windows'
+      # Unix -> bash
+      name: Install CUDA ${{ github.event.inputs.cuda-version }} (Unix)
+      env:
+        cuda: ${{ github.event.inputs.cuda-version }}
+      shell: bash
       run: |
         if [ ! -z ${{ github.event.inputs.cuda-version }} ]; then
           os="$(cut -d'-' -f1 <<< ${{ github.event.inputs.os }})"
@@ -49,7 +69,6 @@ jobs:
             echo "LD_LIBRARY_PATH=${CUDA_PATH}/lib:${LD_LIBRARY_PATH}" >> $GITHUB_ENV
           fi
         fi
-      shell: bash
 
     - name: Set up Python ${{ github.event.inputs.python-version }} on ${{ github.event.inputs.os }}
       uses: actions/setup-python@v2
@@ -62,14 +81,23 @@ jobs:
         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
 
     - name: Install PyTorch ${{ github.event.inputs.pytorch-version }}
+      env:
+        cuda: ${{ github.event.inputs.cuda-version }}
+        torch: ${{ github.event.inputs.pytorch-version }}
+      shell: bash
       run: |
-        version=${{ github.event.inputs.pytorch-version }}
-        if [ ! -z ${{ github.event.inputs.cuda-version }} ]; then
-          cudaversion="$(sed 's/\.//' <<< ${{ github.event.inputs.cuda-version }})"
-          version+="+cu${cudaversion}"
-        else
-          version+="+cpu"
+        os="$(cut -d'-' -f1 <<< ${{ github.event.inputs.os }})"
+        echo "$os"
+        if [ ! -f "./scripts/actions/install_pytorch_${os}.sh" ]; then
+          echo "pytorch not available on ${os}"
+          exit 1
         fi
-        pip install torch=="${version}" -f https://download.pytorch.org/whl/torch_stable.html
+        source "./scripts/actions/install_pytorch_${os}.sh"
+
     - name: Build nitorch
-      run: python setup.py install
+        # Compiling for all architectures takes ages and this workflow is just
+        # here to test that we didn't break anything in the compilation chain
+        # so we only test one architecture (sm_35 -> the default)
+        # In the package distribution workflow, we'll need to compile for
+        # all architectures supported by (the pypi version of) pytorch.
+        run: TORCH_CUDA_ARCH_LIST="3.5" python setup.py install

.github/workflows/test.yml

@@ -1,44 +1,75 @@
 name: nitorch-test
 
 on:
-  workflow_dispatch
-#  push:
-#    branches: [  ]
-#  pull_request:
-#    branches: [  ]
+  workflow_dispatch:
+    inputs:
+      fail-fast:
+        description: 'Fail fast mode'
+        required: false
+        default: 'true'
+  push:
+    paths:
+      # all library files
+      - "nitorch/**"
+    branches: [ master ]
+  pull_request:
+    # always run tests on pull request
+    branches: [ master ]
 
 jobs:
 
-  build:
+  test:
 
     runs-on: ${{ matrix.os }}
     strategy:
-      fail-fast: false
+      fail-fast: ${{ github.event.inputs.fail-fast == 'true' }}
       matrix:
-        os: [ubuntu-latest]
-        python-version: [ 3.6 ]
-        pytorch-version: [ 1.3, 1.6 ]
+        os: [ ubuntu-latest ]               # we only run tests on ubuntu/cpu
+        python-version: [ '3.6' ]           # smallest version supported
+        pytorch-version: [ '1.3', '1.7' ]   # smallest and biggest versions
 
     steps:
+
     - uses: actions/checkout@v2
+
     - name: Set up Python ${{ matrix.python-version }} on ${{ matrix.os }}
       uses: actions/setup-python@v2
       with:
         python-version: ${{ matrix.python-version }}
+
     - name: Install pip
       run: |
         python -m pip install --upgrade pip
         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-    - name: Install PyTorch
-      run: pip install torch==${{ matrix.pytorch-version }}
-    - name: Lint
+        pip install nibabel
+        pip install tifffile
+
+    - name: Install PyTorch ${{ matrix.pytorch-version }}
+      env:
+        cuda: cpu
+        torch: ${{ matrix.pytorch-version }}
+      shell: bash
       run: |
-        pip install flake8
-        flake8 .
-    - name: Build
-      run: python ./setup.py install
-    - name: Test
+        os="$(cut -d'-' -f1 <<< ${{ matrix.os }})"
+        echo "$os"
+        if [ ! -f "./scripts/actions/install_pytorch_${os}.sh" ]; then
+          echo "pytorch not available on ${os}"
+          exit 1
+        fi
+        source "./scripts/actions/install_pytorch_${os}.sh"
+
+    # we must build in development mode for pytest to work in place
+    - name: Build nitorch
+      run: |
+        python setup.py develop
+        python -c "from nitorch._C import spatial"
+
+#    - name: Lint
+#      run: |
+#        pip install flake8
+#        flake8 .
+
+    - name: Tests
       run: |
         pip install pytest
-        pip install pytest-cov
-        pytest --doctest-modules --junitxml=junit/test-results.xml --cov=com --cov-report=xml --cov-report=html
+        pytest

buildtools.py

@@ -73,7 +73,13 @@ def cuda_version():
         return None
     with open(os.devnull, 'w') as devnull:
         version = subprocess.check_output([nvcc, '--version'], stderr=devnull).decode()
-    match = re.search(r'V(?P<version>[0-9\.]+)$', version)
+    match = None
+    for line in version.split('\n'):
+        match = re.search(r'V(?P<version>[0-9\.]+)$', line)
+        if match:
+            break
+    if not match:
+        raise RuntimeError('Failed to parse cuda version')
     version = match.group('version').split('.')
     version = tuple(int(v) for v in version)
     return version

nitorch/io/indexing.py

@@ -82,7 +82,10 @@ def neg2pos(index, shape):
 
     # sanity checks
     try:
-        shape = int(shape)
+        shape0 = shape
+        shape = int(shape0)
+        if shape != shape0:
+            raise TypeError('Shape should be an integer')
     except TypeError:
         raise TypeError('Shape should be an integer')
     if shape < 0:
@@ -97,9 +100,9 @@ def neg2pos(index, shape):
         if index is not None and index < 0:
             index = shape + index
         return index
-    else:
-        assert isinstance(index, (oob_slice, type(None), type(Ellipsis)))
+    elif isinstance(index, (oob_slice, type(None), type(Ellipsis))):
         return index
+    raise TypeError('Index should be an int, slice, Ellipsis or None')
 
 
 def is_fullslice(index, shape, do_neg2pos=True):

nitorch/io/tests/indexing.py → nitorch/io/tests/test_indexing.py

@@ -12,7 +12,7 @@ def test_neg2pos():
     try: idx.neg2pos(0, -1)
     except ValueError: pass
     else: assert False, "raise on negative shape"
-    try:  idx.neg2pos(0, 3.)
+    try:  idx.neg2pos(0, 3.5)
     except TypeError: pass
     else: assert False, "raise on non integer shape"
     try:  idx.neg2pos(4., 3)

nitorch/nn/generators/_mixture.py

@@ -117,7 +117,7 @@ def forward(self, index, **overload):
 
         # check distribution shape
         batches = [batch]
-        channels = [] if channels is 1 else [channels]
+        channels = [] if channels == 1 else [channels]
         for dist in distributions:
             sample_shape = dist.batch_shape + dist.event_shape
             if len(sample_shape) > 0: