From 751e5702e7a50fd31236abf461484440bf1366e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iago=20Su=C3=A1rez?= Date: Sun, 1 Oct 2023 18:27:02 +0200 Subject: [PATCH 1/4] Adding scripts to automatically publish in PyPI --- .github/workflows/build-new.yml | 77 +++++++++++++++++++++++++++++++++ package/build-wheels-linux.sh | 34 +++++++++++++++ package/build-wheels-macos.sh | 62 ++++++++++++++++++++++++++ 3 files changed, 173 insertions(+) create mode 100644 .github/workflows/build-new.yml create mode 100644 package/build-wheels-linux.sh create mode 100644 package/build-wheels-macos.sh diff --git a/.github/workflows/build-new.yml b/.github/workflows/build-new.yml new file mode 100644 index 0000000..aec5632 --- /dev/null +++ b/.github/workflows/build-new.yml @@ -0,0 +1,77 @@ +name: pytlsd build and publish + +on: + push: + branches: + - master + pull_request: + types: [ assigned, opened, synchronize, reopened ] + release: + types: [ published, edited ] + workflow_dispatch: + +jobs: + linux-build: + name: Wrapper Linux Build + runs-on: ubuntu-latest + strategy: + matrix: + platform: [ manylinux2014_x86_64 ] + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + submodules: recursive + - name: Build the Linux wheels + run: | + sudo docker run --rm -e PLAT=${{ matrix.platform }} -v `pwd`:/io quay.io/pypa/${{ matrix.platform }} /io/package/build-wheels-linux.sh + # cleanup for custom runner + sudo chown -R $(whoami):$(whoami) . + - name: Archive wheels + uses: actions/upload-artifact@v3 + with: + # we strip the version number from the artifact name + name: pytlsd-${{ matrix.platform }} + path: wheelhouse/pytlsd-*.whl + + mac-build: + name: Wrapper macOS Build + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ macos-11, macos-12 ] + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + submodules: recursive + - name: Build the macOS wheels + run: | + ./package/build-wheels-macos.sh + - name: Archive wheels + uses: actions/upload-artifact@v3 + with: + name: pytlsd-${{ matrix.os }} + path: ./wheelhouse/pytlsd-*.whl + + pypi-publish: + name: Publish wheels to PyPI + needs: [ linux-build, mac-build ] + runs-on: ubuntu-latest + # We publish the wheel to pypi when a new tag is pushed, + # either by creating a new GitHub release or explictly with `git tag` + if: ${{ github.event_name == 'release' || startsWith(github.ref, 'refs/tags') }} + steps: + - name: Download wheels + uses: actions/download-artifact@v3 + with: + path: ./artifacts/ + - name: Move wheels + run: mkdir ./wheelhouse && mv ./artifacts/**/*.whl ./wheelhouse/ + - name: Publish package + uses: pypa/gh-action-pypi-publish@release/v1 + with: + skip_existing: true + user: __token__ + password: ${{ secrets.PYPI_API_TOKEN }} + packages_dir: ./wheelhouse/ \ No newline at end of file diff --git a/package/build-wheels-linux.sh b/package/build-wheels-linux.sh new file mode 100644 index 0000000..1887fd7 --- /dev/null +++ b/package/build-wheels-linux.sh @@ -0,0 +1,34 @@ +#!/bin/bash +PYTHON_VERSIONS=("cp38-cp38" "cp39-cp39" "cp310-cp310") + +uname -a +echo "Current CentOS Version:" +cat /etc/centos-release + +ls -ltrh /io/ + +CURRDIR=$(pwd) +echo "Num. processes to use for building: $(nproc)" + +# ------ Install dependencies from the default repositories ------ +cd $CURRDIR +yum install -y wget git gcc gcc-c++ cmake make build-essential libopencv-dev + +# ------ Build pytlsd wheel ------ +cd /io/ +WHEEL_DIR="wheels/" +for PYTHON_VERSION in ${PYTHON_VERSIONS[@]}; do + PYTHON_EXEC="/opt/python/${PYTHON_VERSION}/bin/python" + EIGEN3_INCLUDE_DIRS="$EIGEN_DIR" ${PYTHON_EXEC} -m pip wheel --no-deps -w ${WHEEL_DIR} . +done + +PYTHON_DEFAULT="/opt/python/${PYTHON_VERSIONS[-1]}/bin/python" +${PYTHON_DEFAULT} -m pip install auditwheel + +# Bundle external shared libraries into the wheels +OUT_DIR="/io/wheelhouse" +mkdir -p ${OUT_DIR} +for whl in ${WHEEL_DIR}/*.whl; do + auditwheel repair "$whl" -w ${OUT_DIR} --plat ${PLAT} +done +ls -ltrh ${OUT_DIR} diff --git a/package/build-wheels-macos.sh b/package/build-wheels-macos.sh new file mode 100644 index 0000000..c47be52 --- /dev/null +++ b/package/build-wheels-macos.sh @@ -0,0 +1,62 @@ +#!/bin/bash +set -x -e + +PYTHON_VERSIONS=("3.8" "3.9" "3.10") + +# See https://github.com/actions/setup-python/issues/577 +find /usr/local/bin -lname '*/Library/Frameworks/Python.framework/*' -delete +# See https://github.com/actions/setup-python/issues/577#issuecomment-1500828576 +rm /usr/local/bin/go || true +rm /usr/local/bin/gofmt || true + +CURRDIR=$(pwd) +NUM_LOGICAL_CPUS=$(sysctl -n hw.logicalcpu) +echo "Number of logical CPUs is: ${NUM_LOGICAL_CPUS}" + +# Updating requires Xcode 14.0, which cannot be installed on macOS 11. +brew remove swiftlint +brew remove node@18 + +brew update +brew upgrade +brew install wget cmake + +for PYTHON_VERSION in ${PYTHON_VERSIONS[@]}; do + brew install --force "python@${PYTHON_VERSION}" + python${PYTHON_VERSION} -m pip install -U pip setuptools wheel cffi +done + +brew install \ + git \ + cmake \ + opencv + +brew info gcc +brew upgrade gcc +brew info gcc + +# Install `delocate` -- OSX equivalent of `auditwheel` +# see https://pypi.org/project/delocate/ for more details + +cd $CURRDIR +# flags must be passed, to avoid the issue: `Unsupported compiler -- pybind11 requires C++11 support!` +# see https://github.com/quantumlib/qsim/issues/242 for more details +WHEEL_DIR="${CURRDIR}/wheelhouse_unrepaired/" +for PYTHON_VERSION in ${PYTHON_VERSIONS[@]}; do + CC=/usr/local/opt/llvm/bin/clang \ + CXX=/usr/local/opt/llvm/bin/clang++ \ + LDFLAGS=-L/usr/local/opt/libomp/lib \ + python${PYTHON_VERSION} -m pip wheel --no-deps -w ${WHEEL_DIR} . +done + +python${PYTHON_VERSIONS[0]} -m pip install -U delocate + +# Bundle external shared libraries into the wheels +OUT_DIR="${CURRDIR}/wheelhouse" +mkdir -p ${OUT_DIR} +for whl in ${WHEEL_DIR}/*.whl; do + delocate-listdeps --all "$whl" + delocate-wheel -w "${OUT_DIR}" -v "$whl" + rm $whl +done +ls -ltrh ${OUT_DIR} From 160dc832c0a241605cf41b5a66e101ad63025f1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iago=20Su=C3=A1rez?= Date: Sun, 1 Oct 2023 18:31:29 +0200 Subject: [PATCH 2/4] Changing permissions --- package/build-wheels-linux.sh | 0 package/build-wheels-macos.sh | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 package/build-wheels-linux.sh mode change 100644 => 100755 package/build-wheels-macos.sh diff --git a/package/build-wheels-linux.sh b/package/build-wheels-linux.sh old mode 100644 new mode 100755 diff --git a/package/build-wheels-macos.sh b/package/build-wheels-macos.sh old mode 100644 new mode 100755 From 7f0f25dd885c4eeac79efa0cddb80ef19a6d5376 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iago=20Su=C3=A1rez?= Date: Sun, 1 Oct 2023 20:52:50 +0200 Subject: [PATCH 3/4] Changing MAC script --- package/build-wheels-linux.sh | 2 +- package/build-wheels-macos.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package/build-wheels-linux.sh b/package/build-wheels-linux.sh index 1887fd7..c97b04d 100755 --- a/package/build-wheels-linux.sh +++ b/package/build-wheels-linux.sh @@ -19,7 +19,7 @@ cd /io/ WHEEL_DIR="wheels/" for PYTHON_VERSION in ${PYTHON_VERSIONS[@]}; do PYTHON_EXEC="/opt/python/${PYTHON_VERSION}/bin/python" - EIGEN3_INCLUDE_DIRS="$EIGEN_DIR" ${PYTHON_EXEC} -m pip wheel --no-deps -w ${WHEEL_DIR} . + ${PYTHON_EXEC} -m pip wheel --no-deps -w ${WHEEL_DIR} . done PYTHON_DEFAULT="/opt/python/${PYTHON_VERSIONS[-1]}/bin/python" diff --git a/package/build-wheels-macos.sh b/package/build-wheels-macos.sh index c47be52..1678b4d 100755 --- a/package/build-wheels-macos.sh +++ b/package/build-wheels-macos.sh @@ -29,7 +29,7 @@ done brew install \ git \ cmake \ - opencv + gcc brew info gcc brew upgrade gcc From d2c4231a5692e63b1c999d4c313ed17129ea1172 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iago=20Su=C3=A1rez?= Date: Sun, 1 Oct 2023 21:08:49 +0200 Subject: [PATCH 4/4] Using LLVM --- package/build-wheels-macos.sh | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/package/build-wheels-macos.sh b/package/build-wheels-macos.sh index 1678b4d..96cfb09 100755 --- a/package/build-wheels-macos.sh +++ b/package/build-wheels-macos.sh @@ -19,21 +19,17 @@ brew remove node@18 brew update brew upgrade -brew install wget cmake +brew install git wget cmake for PYTHON_VERSION in ${PYTHON_VERSIONS[@]}; do brew install --force "python@${PYTHON_VERSION}" python${PYTHON_VERSION} -m pip install -U pip setuptools wheel cffi done -brew install \ - git \ - cmake \ - gcc - -brew info gcc -brew upgrade gcc -brew info gcc +brew install llvm +brew info llvm +brew upgrade llvm +brew info llvm # Install `delocate` -- OSX equivalent of `auditwheel` # see https://pypi.org/project/delocate/ for more details