Wheels support (#56)

* First try of auto-wheels

* syntax fixes

* syntax fixes -- try2

* dependencies

* fix path

* permissions

* build_tools/build_source.sh

* Debug info

* try3

* moredebuginfo

* moredebug2

* moredebug3

* moredebug4

* typo

* try space

* Files locally

* yumaware

* numpyfix

* testpath

* project

* Cleanup for PR

* Feedback from comments

* Missing pytest update

* Wheels centos (#1)

* Disentangle wheels/dist

* Update build_tools/build_wheels.sh

Co-authored-by: Matthias Feurer <[email protected]>

* Fixed build source

Co-authored-by: Matthias Feurer <[email protected]>

.github/workflows/pytest.yml

@@ -19,18 +19,8 @@ jobs:
         python-version: ${{ matrix.python-version }}
     - name: Install test dependencies
       run: |
-        sudo apt-get install -y build-essential
-        python -m pip install --upgrade pip
-        pip install "numpy<=1.19"
-        sudo apt-get -qq update
-        sudo apt-get install -y libboost-all-dev
-        sudo apt-get remove swig
-        sudo apt-get -y install swig3.0
-        sudo ln -s /usr/bin/swig3.0 /usr/bin/swig
-        sudo gem install coveralls-lcov
-        sudo apt-get install -y lcov
-        sudo apt-get install doxygen
-        pip3 install --user -U pip-tools
+        chmod u+x ./build_tools/env.sh
+        ./build_tools/env.sh
         lcov --directory . --zerocounters
     - name: Setup cmake
       uses: jwlawson/[email protected]

.github/workflows/wheels.yml

@@ -0,0 +1,136 @@
+# Workflow to build and test wheels
+name: Wheel builder
+
+on:
+  push:
+    branches:
+      - master
+      # Release branches
+      - "[0-9]+.[0-9]+.X"
+
+jobs:
+  # Build the wheels for Linux
+  build_wheels:
+    name: Build wheel for cp${{ matrix.python }}-${{ matrix.platform_id }}-${{ matrix.manylinux_image }}
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      # Ensure that a wheel builder finishes even if another fails
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+        python: [36, 37, 38, 39]
+        bitness: [32, 64]
+        manylinux_image: [manylinux2014]
+        include:
+          - os: ubuntu-latest
+            bitness: 64
+            platform_id: manylinux_x86_64
+          - os: ubuntu-latest
+            bitness: 32
+            platform_id: manylinux_i686
+
+    steps:
+      - name: Checkout pyrfr
+        uses: actions/checkout@v1
+
+      - name: Setup Python
+        uses: actions/setup-python@v2
+
+      - name: Build and test wheels
+        env:
+          CIBW_BUILD: cp${{ matrix.python }}-${{ matrix.platform_id }}
+          CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux_image }}
+          CIBW_MANYLINUX_I686_IMAGE: ${{ matrix.manylinux_image }}
+          CIBW_TEST_REQUIRES: pytest threadpoolctl numpy<=1.19
+          CIBW_BEFORE_ALL: "{project}/build_tools/build_wheels.sh"
+          CIBW_TEST_COMMAND: bash {project}/build_tools/test_wheels.sh
+
+        run: |
+          python -m pip install cibuildwheel
+          python -m cibuildwheel --output-dir wheelhouse
+
+      - name: Store artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          path: wheelhouse/*.whl
+
+  # Build the source distribution under Linux
+  build_sdist:
+    name: Source distribution
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout pyrfr
+        uses: actions/checkout@v1
+
+      - name: Setup Python
+        uses: actions/setup-python@v2
+
+      - name: Build source distribution
+        run: bash build_tools/build_source.sh
+
+      - name: Test source distribution
+        run: bash build_tools/test_source.sh
+
+      - name: Store artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          path: dist/*.tar.gz
+
+  # Upload the wheels and the source distribution
+  release_assets:
+    name: Upload Release
+    runs-on: ubuntu-latest
+    needs: [build_wheels, build_sdist]
+    # The artifacts cannot be uploaded on PRs
+    if: github.event_name != 'pull_request'
+
+    steps:
+      - name: Checkout pyrfr
+        uses: actions/checkout@v1
+
+      - name: Download artifacts
+        uses: actions/download-artifact@v2
+        with:
+          path: dist
+
+      - name: Setup Python
+        uses: actions/setup-python@v2
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install setuptools wheel twine
+
+          #- name: Publish python package
+          #  id: publish_package
+          #  env:
+          #    TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
+          #    TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+          #  run: |
+          #    twine upload dist/*
+
+      - name: Create Release
+        id: create_release
+        uses: actions/create-release@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          tag_name: ${{ github.ref }}
+          release_name: Release ${{ github.ref }}
+          draft: false
+          prerelease: false
+      - name: Upload Release Asset
+        id: upload-release-asset
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          python -m pip install --upgrade pip
+          set -x
+          assets=()
+          for asset in ./dist/*/*; do
+            assets+=("-a" "$asset")
+          done
+          tag_name="${GITHUB_REF##*/}"
+          hub release create "${assets[@]}" -m "$tag_name" "$tag_name"

build_tools/build_source.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+set -e
+set -x
+
+# Build the package
+# Install the environment
+chmod u+x ./build_tools/env.sh
+./build_tools/env.sh
+
+# Build the package
+mkdir build
+cd build
+cmake .. && make pyrfr_docstrings
+cd python_package
+
+# Also build the distribution
+python -m pip install twine
+python setup.py sdist -d ../../dist
+
+# Check whether the source distribution will render correctly
+twine check ../../dist/*.tar.gz

build_tools/build_wheels.sh

@@ -0,0 +1,45 @@
+# The wheels are built on a centos docker image
+# To comply with many-linux support, we employ
+# cibuildwheel which build the wheels in a
+# Docker image quay.io with minimal support. Following files are required
+# for the pyrfr to be compiled on the desired target
+pip3 install cmake numpy==1.11.0 scipy==0.17.0
+echo 'echo "pyuic5 $@"' > /usr/local/bin/pyuic5
+chmod +x /usr/local/bin/pyuic5
+yum install -y curl gsl-devel pcre-devel
+curl -LO http://prdownloads.sourceforge.net/swig/swig-3.0.12.tar.gz
+tar xzvf swig-3.0.12.tar.gz
+cd swig-3.0.12
+./configure
+make
+make install
+cd ..
+rm -rf swig-3.0.12*
+swig -version
+
+# Install the package building dependencies -- one line at
+# a time for easy debug -- yum errors out with not much info
+# if one package installation failed
+yum -y install boost
+yum -y install boost-thread
+yum -y install boost-devel
+yum -y install doxygen
+yum -y install openssl-devel
+yum -y install cmake
+yum -y install tree
+yum -y install rsync
+cmake --version
+
+# After installing the dependencies build the python package
+mkdir build
+cd build
+cmake  .. && make pyrfr_docstrings
+# Copy the files for testing
+cp -r ../test_data_sets python_package
+# Copy from /project/build to /project
+rsync -a --delete --exclude '*build*' python_package/ ../
+
+# Wheel building process will create a package from
+# the contents of /project. For debug purposes, show
+# the contents of this directory
+tree /project

build_tools/env.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+set -e
+set -x
+
+# Install the environment requirements
+sudo apt-get install -y build-essential
+python -m pip install --upgrade pip
+pip install "numpy<=1.19"
+sudo apt-get -qq update
+sudo apt-get install -y libboost-all-dev
+sudo apt-get remove swig
+sudo apt-get -y install swig3.0
+sudo ln -s /usr/bin/swig3.0 /usr/bin/swig
+sudo gem install coveralls-lcov
+sudo apt-get install -y lcov
+sudo apt-get install doxygen
+pip3 install --user -U pip-tools

build_tools/test_source.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+set -e
+set -x
+
+# Test the distribution on an isolated venv
+# Directory structure is such that
+# PWD=random_forest_run/random_forest_run/<repo>
+cd ../../
+
+python -m venv test_env
+source test_env/bin/activate
+
+python -m pip install pytest "numpy<=1.19"
+python -m pip install random_forest_run/random_forest_run/dist/*.tar.gz
+
+sed -i -- "s/[^']\+test_data_sets/random_forest_run\/random_forest_run\/test_data_sets/" random_forest_run/random_forest_run/tests/*
+pytest random_forest_run/random_forest_run/tests/*.py

build_tools/test_wheels.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+set -e
+set -x
+
+# The wheel builder will currently have the REPO cloned into
+# /project. Also unittest from pyrfr are compiled and have local
+# paths embedded. Modify such path so they are now pointing to /project
+sed -i -- "s/[^']\+test_data_sets/\/project\/test_data_sets/" /project/tests/*
+pytest -v /project/tests/*py
+
+# Test that there are no links to system libraries
+python -m threadpoolctl -i pyrfr

pyproject.toml

@@ -0,0 +1,6 @@
+[build-system]
+# Minimum requirements for the build system to execute.
+requires = [
+    "setuptools",
+    "wheel",
+]

pyrfr/MANIFEST.in

@@ -1,3 +1,5 @@
 recursive-include include *
 recursive-include pyrfr *.i
-include LICENSE_cereal-1.2.2
+include LICENSE_cereal-1.2.2
+include LICENSE.txt
+include pyproject.toml