Merge pull request #148 from osqp/im/ci

Cleanup CI and Support NumPy 2.0
osqp · Aug 2, 2024 · 350677c · 350677c
2 parents 162c1ae + 2155482
commit 350677c
Show file tree

Hide file tree

Showing 13 changed files with 270 additions and 206 deletions.
diff --git a/.github/workflows/build_aarch64.yml b/.github/workflows/build_aarch64.yml
diff --git a/.github/workflows/build_cuda.yml b/.github/workflows/build_cuda.yml
@@ -38,34 +38,24 @@ jobs:
         package-dir: backend/cuda
         output-dir: wheelhouse
       env:
-        CIBW_BUILD: "cp3?-*"
-        CIBW_SKIP: "*-win32 *-manylinux_i686 *-musllinux_* *-macosx_*"
-        CIBW_TEST_REQUIRES: setuptools pytest torch numdifftools
+        CIBW_BUILD: "cp3*"
+        CIBW_SKIP: "cp36-* cp37-* *-win32 *-manylinux_i686 *-musllinux_* *-macosx_*"
         CIBW_BUILD_VERBOSITY: 1
 
-        CIBW_ENVIRONMENT_LINUX: CMAKE_CUDA_COMPILER=/usr/local/cuda-11.7/bin/nvcc
+        # Clean the build directory between builds
+        CIBW_BEFORE_BUILD: >-
+          rm -rf {package}/osqp_sources/build
+
+        CIBW_ENVIRONMENT_LINUX: CMAKE_CUDA_COMPILER=/usr/local/cuda-12.4/bin/nvcc
         CIBW_BEFORE_ALL_LINUX: bash .github/workflows/prepare_build_environment_linux_cuda.sh
         CIBW_REPAIR_WHEEL_COMMAND_LINUX: ""
 
-        CIBW_ENVIRONMENT_WINDOWS: CMAKE_CUDA_COMPILER="C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v11.6/bin/nvcc.exe" CUDA_TOOLKIT_ROOT_DIR="C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v11.6" CMAKE_GENERATOR_TOOLSET="cuda=C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v11.6"
+        CIBW_ENVIRONMENT_WINDOWS: CMAKE_CUDA_COMPILER="C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v12.5/bin/nvcc.exe" CUDA_TOOLKIT_ROOT_DIR="C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v12.5" CMAKE_GENERATOR_TOOLSET="cuda=C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v12.5"
         CIBW_BEFORE_ALL_WINDOWS: bash .github/workflows/prepare_build_environment_windows_cuda.sh
         CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: ""
 
     - name: Upload artifacts to github
       uses: actions/upload-artifact@v4
       with:
-        name: wheels-${{ matrix.os }}
+        name: wheels-cuda-${{ matrix.os }}
         path: ./wheelhouse
-
-
-  merge_wheels:
-    name: Merge wheel artifacts from build_wheels OS matrix jobs
-    runs-on: ubuntu-latest
-    needs: build_wheels
-    steps:
-      - name: Merge artifacts
-        uses: actions/upload-artifact/merge@v4
-        with:
-          name: wheels
-          pattern: wheels-*
-          delete-merged: true
diff --git a/.github/workflows/build_default.yml b/.github/workflows/build_default.yml
@@ -11,68 +11,99 @@ on:
       - master
 
 jobs:
+  build_sdist:
+    name: Build source
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+    steps:
+    - uses: actions/checkout@master
+      with:
+        submodules: 'recursive'
+
+    - name: Build source
+      run: |
+        python -m pip install build
+        python -m build --sdist --outdir=wheelhouse
+
+    - name: Upload sdist to github
+      uses: actions/upload-artifact@v4
+      with:
+        name: wheels-sdist
+        path: wheelhouse/*.tar.gz
+        if-no-files-found: error
+
   build_wheels:
-    name: Build wheel on ${{ matrix.os }}
+    name: Build wheel on ${{ matrix.os }} for ${{ matrix.cibw_archs }}
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        # Include macos-13 to get Intel x86_64 macs and maos-latest to get the Aaarch64 macs
-        os: [ubuntu-latest, macos-latest, macos-13, windows-2022]
-
-        # Build on the native architectures (macos-latest is arm64. macos-13 is x86_64)
         include:
-        - os: macos-latest
-          osx_arch: 'arm64'
-        - os: macos-13
-          osx_arch: 'x86_64'
+          - os: ubuntu-latest
+            cibw_archs: "x86_64"
+          - os: ubuntu-latest
+            cibw_archs: "aarch64"
+          - os: windows-2022
+            cibw_archs: "auto64"
+          # Include macos-13 to get Intel x86_64 macs and maos-latest to get the Aaarch64 macs
+          - os: macos-13
+            cibw_archs: "x86_64"
+          - os: macos-latest
+            cibw_archs: "arm64"
 
     steps:
     - uses: actions/checkout@master
 
+    # This might not be necessary once ARM runners become available for general use
+    - name: Set up QEMU
+      if: matrix.cibw_archs == 'aarch64'
+      uses: docker/setup-qemu-action@v3
+      with:
+        platforms: arm64
+
     - name: Build wheels
       uses: pypa/[email protected]
       with:
         output-dir: wheelhouse
       env:
-        CIBW_BUILD: "cp3?-*"
-        CIBW_SKIP: "*-win32 *-manylinux_i686 *-musllinux_*"
-        CIBW_TEST_REQUIRES: setuptools pytest torch numdifftools
-        CIBW_TEST_COMMAND: "python -m pytest -s {project}/src/osqp/tests -k \"not codegen\""
-        CIBW_ENVIRONMENT_MACOS: CMAKE_OSX_ARCHITECTURES=${{ matrix.osx_arch }}
+        CIBW_BUILD: "cp3*"
+        CIBW_SKIP: "cp36-* cp37-* *-win32 *-manylinux_i686 *-musllinux_*"
+        # Clean the build directory between builds
+        CIBW_BEFORE_BUILD: >-
+          rm -rf {package}/osqp_sources/build
+        CIBW_TEST_COMMAND: "python -m pytest -s {project}/src/osqp/tests"
+        CIBW_ENVIRONMENT_MACOS: CMAKE_OSX_ARCHITECTURES=${{ matrix.cibw_archs }}
         CIBW_BUILD_VERBOSITY: 1
 
-    - name: Build source
-      if: runner.os == 'Linux'
-      run: |
-        python -m pip install --upgrade build
-        python -m build --sdist --outdir wheelhouse
-
-    - name: Release to pypi
-      if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')
-      env:
-        TWINE_USERNAME: __token__
-        TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
-        TWINE_REPOSITORY: testpypi
-      run: |
-        python -m pip install --upgrade twine
-        twine upload wheelhouse/*
-
     - name: Upload artifacts to github
       uses: actions/upload-artifact@v4
       with:
-        name: wheels-${{ matrix.os }}
-        path: ./wheelhouse
+        name: wheels-${{ runner.os }}-${{ matrix.cibw_archs }}
+        path: ./wheelhouse/*.whl
+        if-no-files-found: error
 
-
-  merge_wheels:
-    name: Merge wheel artifacts from build_wheels OS matrix jobs
+  publish_to_pypi:
+    name: Publish wheels to PyPi
+    if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')
+    needs: [build_sdist, build_wheels]
     runs-on: ubuntu-latest
-    needs: build_wheels
     steps:
-      - name: Merge artifacts
-        uses: actions/upload-artifact/merge@v4
+      - name: Download packages
+        uses: actions/download-artifact@v4
         with:
-          name: wheels
           pattern: wheels-*
-          delete-merged: true
+          path: dist
+          merge-multiple: true
+
+      - name: Print out packages
+        run: ls -la dist/*
+
+      - name: Upload wheels to pypi
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
+          TWINE_REPOSITORY: testpypi
+        run: |
+          python -m pip install --upgrade twine
+          twine upload dist/*
diff --git a/.github/workflows/build_mkl.yml b/.github/workflows/build_mkl.yml
@@ -30,11 +30,14 @@ jobs:
         package-dir: backend/mkl
         output-dir: wheelhouse
       env:
-        CIBW_BUILD: "cp3?-*"
-        CIBW_SKIP: "*-win32 *-manylinux_i686 *-musllinux_*"
-        CIBW_TEST_REQUIRES: setuptools pytest torch numdifftools mkl mkl-devel
+        CIBW_BUILD: "cp3*"
+        CIBW_SKIP: "cp36-* cp37-* *-win32 *-manylinux_i686 *-musllinux_*"
         CIBW_BUILD_VERBOSITY: 1
 
+        # Clean the build directory between builds
+        CIBW_BEFORE_BUILD: >-
+          rm -rf {package}/osqp_sources/build
+
         CIBW_BEFORE_ALL_LINUX: bash .github/workflows/prepare_build_environment_linux_mkl.sh
         CIBW_ENVIRONMENT_LINUX: "MKL_ROOT=/opt/intel/oneapi/mkl/latest"
         CIBW_REPAIR_WHEEL_COMMAND_LINUX: ""
@@ -50,18 +53,5 @@ jobs:
     - name: Upload artifacts to github
       uses: actions/upload-artifact@v4
       with:
-        name: wheels-${{ matrix.os }}
+        name: wheels-mkl-${{ matrix.os }}
         path: ./wheelhouse
-
-
-  merge_wheels:
-    name: Merge wheel artifacts from build_wheels OS matrix jobs
-    runs-on: ubuntu-latest
-    needs: build_wheels
-    steps:
-      - name: Merge artifacts
-        uses: actions/upload-artifact/merge@v4
-        with:
-          name: wheels
-          pattern: wheels-*
-          delete-merged: true
diff --git a/.github/workflows/prepare_build_environment_linux_cuda.sh b/.github/workflows/prepare_build_environment_linux_cuda.sh
@@ -3,9 +3,7 @@
 set -e
 set -x
 
-# Install CUDA 11.2, see:
-# * https://gitlab.com/nvidia/container-images/cuda/-/blob/master/dist/11.2.2/centos7-x86_64/base/Dockerfile
 yum-config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel7/x86_64/cuda-rhel7.repo
-yum install -y cuda-toolkit-11-7
+yum install -y cuda-toolkit-12-4
 
-/usr/local/cuda-11.7/bin/nvcc --version
+/usr/local/cuda-12.4/bin/nvcc --version
diff --git a/.github/workflows/prepare_build_environment_windows_cuda.sh b/.github/workflows/prepare_build_environment_windows_cuda.sh
@@ -3,6 +3,6 @@
 set -e
 set -x
 
-curl -s -L -nv -o cuda.exe https://developer.download.nvidia.com/compute/cuda/11.6.1/local_installers/cuda_11.6.1_511.65_windows.exe
-./cuda.exe -s nvcc_11.6 cudart_11.6 cublas_dev_11.6 curand_dev_11.6 cusparse_dev_11.6 thrust_11.6 visual_studio_integration_11.6
+curl -s -L -nv -o cuda.exe https://developer.download.nvidia.com/compute/cuda/12.5.1/local_installers/cuda_12.5.1_555.85_windows.exe
+./cuda.exe -s nvcc_12.5 cudart_12.5 cublas_dev_12.5 curand_dev_12.5 cusparse_dev_12.5 thrust_12.5 visual_studio_integration_12.5
 rm cuda.exe
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -21,7 +21,7 @@ list(APPEND CMAKE_MESSAGE_INDENT "  ")
 FetchContent_Declare(
   osqp
   GIT_REPOSITORY https://github.com/osqp/osqp.git
-  GIT_TAG 02a117cfc8ad21b06c2596603a2046ee61c82786
+  GIT_TAG ff371613bef82e85fb431590343853846ff96203
 )
 list(POP_BACK CMAKE_MESSAGE_INDENT)
 FetchContent_MakeAvailable(osqp)

diff --git a/pyproject.toml b/pyproject.toml
@@ -16,10 +16,7 @@ dependencies = [
     "jinja2",
     "numpy>=1.7",
     "qdldl",
-    # Exclude scipy 1.12 because the random sparse array function started returning
-    # the transpose of the original, breaking the unit tests. This was fixed in 1.13.0.
-    # ref: https://github.com/scipy/scipy/issues/20027
-    "scipy>=0.13.2,!=1.12.0",
+    "scipy>=0.13.2",
     "setuptools",
     "joblib",
 ]
@@ -29,10 +26,20 @@ cuda = [
     "osqp-cuda",
 ]
 dev = [
-    "numdifftools",
     "pre-commit",
     "pytest>=6",
     "torch",
+
+    # Exclude scipy 1.12 because the random sparse array function started returning
+    # the transpose of the original, breaking the unit tests. This was fixed in 1.13.0.
+    # This shouldn't actually affect the users, so there shouldn't be a need to exclude
+    # 1.12 on a user's machine.
+    # ref: https://github.com/scipy/scipy/issues/20027
+    "scipy!=1.12.0",
+
+    # As of 07/27/24, torch on many platforms (intel macs, Windows), cannot seem to coexist
+    # with numpy>=2; Since we do need torch for development purposes, we'll pin np<2
+    "numpy<2"
 ]
 mkl = [
     "osqp-mkl",
@@ -58,5 +65,9 @@ CMAKE_OSX_ARCHITECTURES = {env="CMAKE_OSX_ARCHITECTURES"}
 [tool.pytest.ini_options]
 testpaths = ["src/osqp/tests"]
 
+[tool.cibuildwheel]
+# Install the dependencies needed when testing the built wheels
+test-extras = ["dev"]
+
 [tool.setuptools_scm]
 write_to = "src/osqp/_version.py"