From e96b53253fd1b4ff265fdaf90d5eb6436b771584 Mon Sep 17 00:00:00 2001 From: nmreadelf Date: Wed, 26 Jul 2023 11:23:35 +0800 Subject: [PATCH] Added arm build --- .github/workflows/build_arm_whells.yml | 397 +++++++++++++++++++++++++ 1 file changed, 397 insertions(+) create mode 100644 .github/workflows/build_arm_whells.yml diff --git a/.github/workflows/build_arm_whells.yml b/.github/workflows/build_arm_whells.yml new file mode 100644 index 00000000000..b89f6cca01a --- /dev/null +++ b/.github/workflows/build_arm_whells.yml @@ -0,0 +1,397 @@ +name: Build + +on: + push: + tags: + - 'v*' + branches: + - main + paths-ignore: + - '**.md' + - '**.yml' + - '.github/**' + - 'examples/**' + - 'tests/**' + pull_request: + branches: + - main + workflow_dispatch: + +jobs: + build_wheels_linux: + name: ${{ matrix.os }} py${{ matrix.python-version }} + runs-on: + - self-hosted + - ARM64 + strategy: + fail-fast: false + matrix: + os: [ ubuntu-20.04 ] + python-version: [ "3.8", "3.9", "3.10", "3.11"] + # python-version: [ "3.7" ] + env: + RUNNER_OS: ${{ matrix.os }} + PYTHON_VERSION: ${{ matrix.python-version }} + steps: + - name: switch python + run: | + pyenv local "${{ matrix.python-version }}" + - name: Install clang++ for Ubuntu + if: matrix.os == 'ubuntu-20.04' + run: | + pwd + uname -a + wget https://apt.llvm.org/llvm.sh + chmod +x llvm.sh + sudo ./llvm.sh 15 + which clang++-15 + clang++-15 --version + sudo apt-get install -y make cmake ccache ninja-build yasm gawk wget + ccache -s + - name: Update git + run: | + sudo add-apt-repository ppa:git-core/ppa -y + sudo apt-get update + sudo apt-get install -y git + git --version + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Restore submodules cache + uses: actions/cache/restore@v3 + id: cache + with: + path: | + contrib/** + key: | + submodule-${{ hashFiles('.gitmodules') }} + - name: Update submodules if cache miss + if: steps.cache.outputs.cache-hit != 'true' + run: | + git submodule update --init --recursive --jobs 4 + - name: Save submodules cache + if: steps.cache.outputs.cache-hit != 'true' + uses: actions/cache/save@v3 + with: + path: | + contrib/** + key: | + submodule-${{ hashFiles('.gitmodules') }} + - name: ccache + uses: hendrikmuhs/ccache-action@v1.2 + with: + key: ${{ matrix.os }} + max-size: 5G + append-timestamp: true + - name: remove old clang and link clang-15 to clang + if: matrix.os == 'ubuntu-20.04' + run: | + sudo rm /usr/bin/clang + sudo ln -s /usr/bin/clang-15 /usr/bin/clang + sudo rm /usr/bin/clang++ + sudo ln -s /usr/bin/clang++-15 /usr/bin/clang++ + which clang++ + clang++ --version + - name: Run chdb/build.sh + run: | + python3 -m pip install pybind11 + export CC=/usr/bin/clang + export CXX=/usr/bin/clang++ + bash ./chdb/build.sh + python3 -m pip install pandas pyarrow + bash -x ./chdb/test_smoke.sh + continue-on-error: false + - name: Check ccache statistics + run: | + ccache -s + ls -lh chdb + df -h + - name: Install dependencies for building wheels + run: python3 -m pip install -U pip tox pybind11 twine setuptools wheel + - name: Build wheels + run: | + export CC=/usr/bin/clang + export CXX=/usr/bin/clang++ + make wheel + - name: Install patchelf from github + run: | + wget https://github.com/NixOS/patchelf/releases/download/0.17.2/patchelf-0.17.2-x86_64.tar.gz -O patchelf.tar.gz + tar -xvf patchelf.tar.gz + sudo cp bin/patchelf /usr/bin/ + sudo chmod +x /usr/bin/patchelf + patchelf --version + - name: Audit wheels + run: | + python3 -m pip install auditwheel + auditwheel -v repair -w dist/ --plat manylinux2014_x86_64 dist/*.whl + continue-on-error: false + - name: Show files + run: | + sudo rm -f dist/*-linux_x86_64.whl + ls -lh dist + shell: bash + - name: Run tests + run: | + python3 -m pip install dist/*.whl + python3 -m pip install pandas pyarrow psutil + python3 -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(res)" + make test + continue-on-error: false + - uses: actions/upload-artifact@v3 + with: + path: ./dist/*.whl + - name: Upload pypi + if: startsWith(github.ref, 'refs/tags/v') + run: | + python3 -m pip install twine + python3 -m twine upload dist/*.whl + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + build_wheels_macos_12: + name: ${{ matrix.os }} py${{ matrix.python-version }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ macos-12 ] + # python-version: [ "3.8", "3.9", "3.10", "3.11"] + python-version: [ "3.11" ] + env: + RUNNER_OS: ${{ matrix.os }} + PYTHON_VERSION: ${{ matrix.python-version }} + steps: + - name: Install python + uses: actions/setup-python@v4 + with: + python-version: "${{ matrix.python-version }}" + - name: Remove /usr/local/bin/python3 + run: | + sudo rm -f /usr/local/bin/python3 + - name: Install clang++ for macOS + run: | + pwd + uname -a + export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 + brew update + brew install git ccache ninja libtool gettext llvm@15 gcc binutils grep findutils zstd + export PATH=$(brew --prefix llvm@15)/bin:$PATH + which clang++ + clang++ --version + ccache -s + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Restore submodules cache + uses: actions/cache/restore@v3 + id: cache + with: + path: | + contrib/** + key: | + submodule-${{ hashFiles('.gitmodules') }} + - name: Update submodules if cache miss + if: steps.cache.outputs.cache-hit != 'true' + run: | + git submodule update --init --recursive --jobs 4 + - name: Save submodules cache + if: steps.cache.outputs.cache-hit != 'true' + uses: actions/cache/save@v3 + with: + path: | + contrib/** + key: | + submodule-${{ hashFiles('.gitmodules') }} + - name: ccache + uses: hendrikmuhs/ccache-action@v1.2 + with: + key: ${{ matrix.os }} + max-size: 5G + append-timestamp: true + - name: Run chdb/build.sh + timeout-minutes: 300 + run: | + python3 -m pip install pybind11 + export PATH=$(brew --prefix llvm@15)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin + export CC=$(brew --prefix llvm@15)/bin/clang + export CXX=$(brew --prefix llvm@15)/bin/clang++ + bash gen_manifest.sh + bash ./chdb/build.sh + python3 -m pip install pandas pyarrow + bash -x ./chdb/test_smoke.sh + continue-on-error: false + - name: Keep killall ccache and wait for ccache to finish + if: always() + run: | + sleep 60 + while ps -ef | grep ccache | grep -v grep; do \ + killall ccache; \ + sleep 10; \ + done + - name: Check ccache statistics + run: | + ccache -s + ls -lh chdb + df -h + env: + CIBW_ENVIRONMENT_MACOS: "PATH=$(brew --prefix llvm@15)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin CC=$(brew --prefix llvm@15)/bin/clang CXX=$(brew --prefix llvm@15)/bin/clang++" + - name: Install dependencies for building wheels + run: | + python3 -m pip install -U pip tox pybind11 twine setuptools wheel>=0.40.0 + - name: Build wheels + run: | + export PATH=$(brew --prefix llvm@15)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin + export CC=$(brew --prefix llvm@15)/bin/clang + export CXX=$(brew --prefix llvm@15)/bin/clang++ + make wheel + - name: Fix wheel platform tag + run: | + python3 -m wheel tags --platform-tag=macosx_10_15_x86_64 --remove dist/*.whl + - name: Run tests + run: | + python3 -m pip install dist/*.whl + python3 -m pip install pandas pyarrow psutil + python3 -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(res)" + make test + continue-on-error: false + - name: Show files + run: ls -lh dist + shell: bash + - uses: actions/upload-artifact@v3 + with: + path: ./dist/*.whl + - name: Upload pypi + if: startsWith(github.ref, 'refs/tags/v') + run: | + python3 -m pip install twine + python3 -m twine upload dist/*.whl + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + + build_wheels_macos: + name: cbw on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ macos-11 ] + # python-version: [ "3.8", "3.9", "3.10"] + python-version: [ "3.11" ] + env: + RUNNER_OS: ${{ matrix.os }} + PYTHON_VERSION: ${{ matrix.python-version }} + steps: + - name: Install clang++ for macOS + if: matrix.os == 'macos-11' || matrix.os == 'macos-12' + run: | + pwd + uname -a + export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 + brew update + brew install git ccache ninja libtool gettext llvm@15 gcc binutils grep findutils zstd + export PATH=$(brew --prefix llvm@15)/bin:$PATH + which clang++ + clang++ --version + ccache -s + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Restore submodules cache + uses: actions/cache/restore@v3 + id: cache + with: + path: | + contrib/** + key: | + submodule-${{ hashFiles('.gitmodules') }} + - name: Update submodules if cache miss + if: steps.cache.outputs.cache-hit != 'true' + run: | + git submodule update --init --recursive --jobs 4 + - name: ccache + uses: hendrikmuhs/ccache-action@v1.2 + with: + key: ${{ matrix.os }} + max-size: 5G + append-timestamp: true + - name: Prepare chdb/build.sh + run: | + python3 -m pip install pybind11 + export PATH=$(brew --prefix llvm@15)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin + export CC=$(brew --prefix llvm@15)/bin/clang + export CXX=$(brew --prefix llvm@15)/bin/clang++ + continue-on-error: false + - name: Check ccache statistics + run: | + ccache -s + ls -lh chdb + df -h + env: + CIBW_ENVIRONMENT_MACOS: "PATH=$(brew --prefix llvm@15)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin CC=$(brew --prefix llvm@15)/bin/clang CXX=$(brew --prefix llvm@15)/bin/clang++" + - name: Remove /usr/local/bin/python3 + run: | + sudo rm -f /usr/local/bin/python3 + - name: Install dependencies for building wheels + run: | + python3 -m pip install -U pip tox pybind11 twine setuptools wheel + python3 -m pip install cibuildwheel==2.12.1 + - name: Build wheels + run: python3 -m cibuildwheel --output-dir wheelhouse + timeout-minutes: 300 + env: + CIBW_ENVIRONMENT_MACOS: "PATH=$(brew --prefix llvm@15)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin CC=$(brew --prefix llvm@15)/bin/clang CXX=$(brew --prefix llvm@15)/bin/clang++" + CIBW_DEBUG: 1 + CIBW_BEFORE_BUILD: "pip install -U pip tox pybind11 && bash -x gen_manifest.sh && bash chdb/build.sh" + CIBW_BUILD_VERBOSITY: 3 + CIBW_BUILD: "cp38-macosx_x86_64 cp39-macosx_x86_64 cp310-macosx_x86_64" + CIBW_TEST_REQUIRES: "pyarrow pandas psutil" + CIBW_TEST_COMMAND: "cd {project} && make test" + - name: Keep killall ccache and wait for ccache to finish + if: always() + run: | + sleep 60 + while ps -ef | grep ccache | grep -v grep; do \ + killall ccache; \ + sleep 10; \ + done + - name: Show files + run: ls -lh wheelhouse + shell: bash + - uses: actions/upload-artifact@v3 + with: + path: ./wheelhouse/*.whl + - name: Upload pypi + if: startsWith(github.ref, 'refs/tags/v') + run: | + python3 -m pip install twine + python3 -m twine upload wheelhouse/*.whl + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + + # #build for macos12 arm64(Apple Silicon) + # build_wheels_macos_arm64: + + ## Experimental Restart on Expire/Cancel (WIP) + ## Requires: PAT Token + ## the PAT token should have permissions according to this + ## documentation - https://developer.github.com/v3/actions/workflow-runs/ + restart_failed: + name: Retry Expired Builds + runs-on: ubuntu-latest + if: github.event_name == 'push' && cancelled() + needs: [ build_wheels_linux, build_wheels_macos, build_wheels_macos_12 ] + steps: + - name: Sleep for 20 seconds + run: sleep 20s + shell: bash + - name: Retry Failed/Expired Build + run: | + curl -i \ + -X POST \ + -H "Accept: application/vnd.github.v3+json" \ + -H "Authorization: token ${{ secrets.PAT_TOKEN }}" \ + https://api.github.com/repos/chdb-io/chdb/actions/runs/${{ github.run_id }}/rerun \ + -d '{"ref": "${{ github.ref }}" }' +