From ddbb9b766f0ab4a1c8c7b1c9d73138a75a1ac7c2 Mon Sep 17 00:00:00 2001 From: nmreadelf Date: Wed, 26 Jul 2023 11:23:35 +0800 Subject: [PATCH 1/4] Add arm build --- .github/workflows/build_arm_whells.yml | 156 +++++++++++++++++++++++++ 1 file changed, 156 insertions(+) create mode 100644 .github/workflows/build_arm_whells.yml diff --git a/.github/workflows/build_arm_whells.yml b/.github/workflows/build_arm_whells.yml new file mode 100644 index 00000000000..fafe77288d3 --- /dev/null +++ b/.github/workflows/build_arm_whells.yml @@ -0,0 +1,156 @@ +name: Build + +on: + push: + tags: + - 'v*' + branches: + - main + paths-ignore: + - '**.md' + - '**.yml' + - '.github/**' + - 'examples/**' + - 'tests/**' + pull_request: + branches: + - main + workflow_dispatch: + +defaults: + run: + shell: bash -leo pipefail {0} + +jobs: + build_wheels_linux_arm64: + name: ${{ matrix.os }} py${{ matrix.python-version }} + runs-on: + - self-hosted + - ARM64 + strategy: + fail-fast: false + matrix: + os: [ ubuntu-20.04 ] + # python-version: [ "3.8", "3.9", "3.10", "3.11"] + python-version: [ "3.11" ] + env: + RUNNER_OS: ${{ matrix.os }} + PYTHON_VERSION: ${{ matrix.python-version }} + steps: + - name: switch python + run: | + export PYENV_ROOT="$HOME/.pyenv" + export PATH="$PYENV_ROOT/bin:$PATH" + pyenv local "${{ matrix.python-version }}" + python --version + shell: bash -l {0} + - name: Update git + run: | + sudo add-apt-repository ppa:git-core/ppa -y + sudo apt-get update + sudo apt-get install -y git + git --version + shell: bash -l {0} + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Restore submodules cache + uses: actions/cache/restore@v3 + id: cache + with: + path: | + contrib/** + key: | + submodule-${{ hashFiles('.gitmodules') }} + - name: Update submodules if cache miss + if: steps.cache.outputs.cache-hit != 'true' + run: | + git submodule update --init --recursive --jobs 4 + - name: Save submodules cache + if: steps.cache.outputs.cache-hit != 'true' + uses: actions/cache/save@v3 + with: + path: | + contrib/** + key: | + submodule-${{ hashFiles('.gitmodules') }} + - name: ccache + uses: hendrikmuhs/ccache-action@v1.2 + with: + key: ${{ matrix.os }} + max-size: 5G + append-timestamp: true + - name: Prepare chdb/build.sh + run: | + pyenv local "${{ matrix.python-version }}" + python3 --version + export CC=/usr/bin/clang + export CXX=/usr/bin/clang++ + shell: bash -l {0} + continue-on-error: false + - name: Install dependencies for building wheels + run: | + python3 -m pip install -U pip tox pybind11 twine setuptools wheel + python3 -m pip install cibuildwheel==2.12.1 + shell: bash -l {0} + - name: Build wheels + run: python3 -m cibuildwheel --output-dir wheelhouse + timeout-minutes: 300 + env: + CIBW_DEBUG: 1 + CIBW_BEFORE_BUILD_LINUX: "yum install -y wget && wget https://apt.llvm.org/llvm.sh; chmod +x llvm.sh; bash ./llvm.sh 15; which clang++-15; wget https://github.com/NixOS/patchelf/releases/download/0.18.0/patchelf-0.18.0-aarch64.tar.gz -O patchelf.tar.gz; tar -xvf patchelf.tar.gz; cp bin/patchelf /usr/bin/; chmod +x /usr/bin/patchelf; patchelf --version;" + CIBW_CONTAINER_ENGINE: podman + CIBW_ENVIRONMENT_LINUX: "CC=/usr/bin/clang CXX=/usr/bin/clang++" + CIBW_BEFORE_BUILD: "pip install -U pip tox pybind11 && bash -x gen_manifest.sh && bash chdb/build.sh" + CIBW_BUILD_VERBOSITY: 3 + CIBW_BUILD: "cp38-manylinux_aarch64 cp39-manylinux_aarch64 cp310-manylinux_aarch64" + CIBW_TEST_REQUIRES: "pyarrow pandas psutil" + CIBW_TEST_COMMAND: "cd {project} && make test" + - name: Keep killall ccache and wait for ccache to finish + if: always() + run: | + sleep 60 + while ps -ef | grep ccache | grep -v grep; do \ + killall ccache; \ + sleep 10; \ + done + - name: Show files + run: ls -lh wheelhouse + shell: bash + - uses: actions/upload-artifact@v3 + with: + path: ./wheelhouse/*.whl + - name: Upload pypi + if: startsWith(github.ref, 'refs/tags/v') + run: | + python3 -m pip install twine + python3 -m twine upload wheelhouse/*.whl + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + + # #build for macos12 arm64(Apple Silicon) + # build_wheels_macos_arm64: + + ## Experimental Restart on Expire/Cancel (WIP) + ## Requires: PAT Token + ## the PAT token should have permissions according to this + ## documentation - https://developer.github.com/v3/actions/workflow-runs/ + restart_failed: + name: Retry Expired Builds + runs-on: ubuntu-latest + if: github.event_name == 'push' && cancelled() + needs: [ build_wheels_linux_arm64 ] + steps: + - name: Sleep for 20 seconds + run: sleep 20s + shell: bash + - name: Retry Failed/Expired Build + run: | + curl -i \ + -X POST \ + -H "Accept: application/vnd.github.v3+json" \ + -H "Authorization: token ${{ secrets.PAT_TOKEN }}" \ + https://api.github.com/repos/chdb-io/chdb/actions/runs/${{ github.run_id }}/rerun \ + -d '{"ref": "${{ github.ref }}" }' + From cb8d3dc5abc16d6aa85968b31c60ff886af92c0c Mon Sep 17 00:00:00 2001 From: Lorenzo Mangani Date: Thu, 27 Jul 2023 16:25:52 +0200 Subject: [PATCH 2/4] Update build_arm_whells.yml --- .github/workflows/build_arm_whells.yml | 44 ++++---------------------- 1 file changed, 7 insertions(+), 37 deletions(-) diff --git a/.github/workflows/build_arm_whells.yml b/.github/workflows/build_arm_whells.yml index fafe77288d3..1bd6f1a5a86 100644 --- a/.github/workflows/build_arm_whells.yml +++ b/.github/workflows/build_arm_whells.yml @@ -1,21 +1,13 @@ -name: Build +name: Build arm64 on: - push: - tags: - - 'v*' - branches: - - main - paths-ignore: - - '**.md' - - '**.yml' - - '.github/**' - - 'examples/**' - - 'tests/**' - pull_request: - branches: - - main workflow_dispatch: + inputs: + TAG_NAME: + description: 'Release Version Tag' + required: true + release: + types: [created] defaults: run: @@ -132,25 +124,3 @@ jobs: # #build for macos12 arm64(Apple Silicon) # build_wheels_macos_arm64: - ## Experimental Restart on Expire/Cancel (WIP) - ## Requires: PAT Token - ## the PAT token should have permissions according to this - ## documentation - https://developer.github.com/v3/actions/workflow-runs/ - restart_failed: - name: Retry Expired Builds - runs-on: ubuntu-latest - if: github.event_name == 'push' && cancelled() - needs: [ build_wheels_linux_arm64 ] - steps: - - name: Sleep for 20 seconds - run: sleep 20s - shell: bash - - name: Retry Failed/Expired Build - run: | - curl -i \ - -X POST \ - -H "Accept: application/vnd.github.v3+json" \ - -H "Authorization: token ${{ secrets.PAT_TOKEN }}" \ - https://api.github.com/repos/chdb-io/chdb/actions/runs/${{ github.run_id }}/rerun \ - -d '{"ref": "${{ github.ref }}" }' - From 1f135f57619f8c494de85daa90531affd249aab1 Mon Sep 17 00:00:00 2001 From: nmreadelf Date: Fri, 4 Aug 2023 08:08:31 +0800 Subject: [PATCH 3/4] fix build wheel --- ...ld_arm_whells.yml => build_arm_wheels.yml} | 95 +++++++++++-------- .github/workflows/build_wheels.yml | 22 ++--- Makefile | 7 +- chdb/build_linux_arm64.sh | 58 +++++++++++ {packages => chdb}/build_mac_arm64.sh | 0 5 files changed, 125 insertions(+), 57 deletions(-) rename .github/workflows/{build_arm_whells.yml => build_arm_wheels.yml} (53%) create mode 100755 chdb/build_linux_arm64.sh rename {packages => chdb}/build_mac_arm64.sh (100%) diff --git a/.github/workflows/build_arm_whells.yml b/.github/workflows/build_arm_wheels.yml similarity index 53% rename from .github/workflows/build_arm_whells.yml rename to .github/workflows/build_arm_wheels.yml index 1bd6f1a5a86..1cff25ea235 100644 --- a/.github/workflows/build_arm_whells.yml +++ b/.github/workflows/build_arm_wheels.yml @@ -1,6 +1,12 @@ name: Build arm64 on: + # for test action + pull_request: + branches: + - main + - feat/add-arm-wheel-build + # end for test action workflow_dispatch: inputs: TAG_NAME: @@ -23,19 +29,22 @@ jobs: fail-fast: false matrix: os: [ ubuntu-20.04 ] - # python-version: [ "3.8", "3.9", "3.10", "3.11"] - python-version: [ "3.11" ] env: RUNNER_OS: ${{ matrix.os }} - PYTHON_VERSION: ${{ matrix.python-version }} + PYTHON_VERSIONS: "3.8 3.9 3.10 3.11" steps: - - name: switch python + - name: Install clang++ for Ubuntu + if: matrix.os == 'ubuntu-20.04' run: | - export PYENV_ROOT="$HOME/.pyenv" - export PATH="$PYENV_ROOT/bin:$PATH" - pyenv local "${{ matrix.python-version }}" - python --version - shell: bash -l {0} + pwd + uname -a + wget https://apt.llvm.org/llvm.sh + chmod +x llvm.sh + sudo ./llvm.sh 15 + which clang++-15 + clang++-15 --version + sudo apt-get install -y make cmake ccache ninja-build yasm gawk wget + ccache -s - name: Update git run: | sudo add-apt-repository ppa:git-core/ppa -y @@ -72,49 +81,53 @@ jobs: key: ${{ matrix.os }} max-size: 5G append-timestamp: true - - name: Prepare chdb/build.sh + - name: remove old clang and link clang-15 to clang + if: matrix.os == 'ubuntu-20.04' run: | - pyenv local "${{ matrix.python-version }}" - python3 --version - export CC=/usr/bin/clang - export CXX=/usr/bin/clang++ - shell: bash -l {0} + sudo rm /usr/bin/clang + sudo ln -s /usr/bin/clang-15 /usr/bin/clang + sudo rm /usr/bin/clang++ + sudo ln -s /usr/bin/clang++-15 /usr/bin/clang++ + which clang++ + clang++ --version + - name: Make linux-arm64 + run: | + #make linux-arm64 + bash -l -e chdb/build_linux_arm64.sh continue-on-error: false - - name: Install dependencies for building wheels + - name: Check ccache statistics run: | - python3 -m pip install -U pip tox pybind11 twine setuptools wheel - python3 -m pip install cibuildwheel==2.12.1 - shell: bash -l {0} - - name: Build wheels - run: python3 -m cibuildwheel --output-dir wheelhouse - timeout-minutes: 300 - env: - CIBW_DEBUG: 1 - CIBW_BEFORE_BUILD_LINUX: "yum install -y wget && wget https://apt.llvm.org/llvm.sh; chmod +x llvm.sh; bash ./llvm.sh 15; which clang++-15; wget https://github.com/NixOS/patchelf/releases/download/0.18.0/patchelf-0.18.0-aarch64.tar.gz -O patchelf.tar.gz; tar -xvf patchelf.tar.gz; cp bin/patchelf /usr/bin/; chmod +x /usr/bin/patchelf; patchelf --version;" - CIBW_CONTAINER_ENGINE: podman - CIBW_ENVIRONMENT_LINUX: "CC=/usr/bin/clang CXX=/usr/bin/clang++" - CIBW_BEFORE_BUILD: "pip install -U pip tox pybind11 && bash -x gen_manifest.sh && bash chdb/build.sh" - CIBW_BUILD_VERBOSITY: 3 - CIBW_BUILD: "cp38-manylinux_aarch64 cp39-manylinux_aarch64 cp310-manylinux_aarch64" - CIBW_TEST_REQUIRES: "pyarrow pandas psutil" - CIBW_TEST_COMMAND: "cd {project} && make test" - - name: Keep killall ccache and wait for ccache to finish - if: always() + ccache -s + ls -lh chdb + df -h + - name: Install patchelf from github run: | - sleep 60 - while ps -ef | grep ccache | grep -v grep; do \ - killall ccache; \ - sleep 10; \ - done + pyenv local "${{ matrix.python-version }}" + wget https://github.com/NixOS/patchelf/releases/download/0.18.0/patchelf-0.18.0-aarch64.tar.gz -O patchelf.tar.gz + tar -xvf patchelf.tar.gz + sudo cp bin/patchelf /usr/bin/ + sudo chmod +x /usr/bin/patchelf + patchelf --version + - name: Audit wheels + run: | + pyenv local "${{ matrix.python-version }}" + python3 -m pip install auditwheel + auditwheel -v repair -w dist/ --plat manylinux_2_17_aarch64 dist/*.whl + continue-on-error: false - name: Show files - run: ls -lh wheelhouse + run: | + # e.g: remove chdb-0.11.4-cp310-cp310-linux_aarch64.whl, keep chdb-0.11.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl + sudo rm -f dist/*linux_aarch64.whl + ls -lh dist shell: bash + continue-on-error: false - uses: actions/upload-artifact@v3 with: - path: ./wheelhouse/*.whl + path: ./dist/*.whl - name: Upload pypi if: startsWith(github.ref, 'refs/tags/v') run: | + pyenv local "${{ matrix.python-version }}" python3 -m pip install twine python3 -m twine upload wheelhouse/*.whl env: diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index f4a1e51a776..3bd49b0a588 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -1,21 +1,13 @@ -name: Build +name: Build X86 on: - push: - tags: - - 'v*' - branches: - - main - paths-ignore: - - '**.md' - - '**.yml' - - '.github/**' - - 'examples/**' - - 'tests/**' - pull_request: - branches: - - main workflow_dispatch: + inputs: + TAG_NAME: + description: 'Release Version Tag' + required: true + release: + types: [created] jobs: build_wheels_linux: diff --git a/Makefile b/Makefile index f300ca810aa..bcc80cf3429 100644 --- a/Makefile +++ b/Makefile @@ -26,7 +26,12 @@ clean: mac-arm64: @echo "Make macOS arm64 whl" - packages/build_mac_arm64.sh + chdb/build_mac_arm64.sh + @echo "Done." + +linux-arm64: + @echo "Make linux arm64 whl" + chdb/build_linux_arm64.sh @echo "Done." build: clean buildlib wheel diff --git a/chdb/build_linux_arm64.sh b/chdb/build_linux_arm64.sh new file mode 100755 index 00000000000..410249ceb81 --- /dev/null +++ b/chdb/build_linux_arm64.sh @@ -0,0 +1,58 @@ +#!/bin/bash -e +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +PROJ_DIR=$(dirname ${DIR}) + +[ -z "${PYTHON_VERSIONS}" ] && { echo "please provide PYTHON_VERSIONS env, e.g: PYTHON_VERSIONS='3.8 3.9'"; exit 1; } + +# echo ${PROJ_DIR} + +for PY_VER in ${PYTHON_VERSIONS}; do + cd ${PROJ_DIR} + pyenv local "${PY_VER}" + python3 --version + python3 -m pip install pybind11 + export CC=/usr/bin/clang + export CXX=/usr/bin/clang++ + # Install universal2 pkg + python3 -VV + python3-config --includes + # if python3 -VV does not contain ${PY_VER}, then exit + if ! python3 -VV 2>&1 | grep -q "${PY_VER}"; then + echo "Error: Required version of Python (${PY_VER}) not found. Aborting." + exit 1 + fi + + python3 -m pip install -U pybind11 wheel build tox + rm -rf ./buildlib + + ./chdb/build.sh + cd chdb && python3 -c "import _chdb; res = _chdb.query('select 1112222222,555', 'JSON'); print(res)" && cd - + + ./gen_manifest.sh + cat ./MANIFEST.in + + # try delete + whl_file=$(find dist | grep 'whl$' | grep cp${PY_VER//./}-cp${PY_VER//./} || echo "notfound.whl") + rm -f ${whl_file} || : + + python3 -m build --wheel + + python3 -m wheel tags --platform-tag=manylinux_2_17_aarch64 + + find dist + + python3 -m pip install pandas pyarrow psutil + find dist + whl_file=$(find dist | grep 'whl$' | grep cp${PY_VER//./}-cp${PY_VER//./}) + python3 -m pip install --force-reinstall ${whl_file} + + python3 -c "import chdb; res = chdb.query('select version()', 'CSV'); print(res)" + + python3 -m chdb "SELECT 1, 'ab'" arrowtable + + python3 -m chdb "SELECT 1, 'ab'" dataframe + + bash -x ./chdb/test_smoke.sh + + make test +done diff --git a/packages/build_mac_arm64.sh b/chdb/build_mac_arm64.sh similarity index 100% rename from packages/build_mac_arm64.sh rename to chdb/build_mac_arm64.sh From 172aca105ddaa148e86366f6a0c04d6ab1f5bbea Mon Sep 17 00:00:00 2001 From: Lorenzo Mangani Date: Sun, 6 Aug 2023 11:36:03 +0200 Subject: [PATCH 4/4] Cleanup Job Cleanup Job for Self-Hosted runners to reclaim disk space after successful or failed builds. --- .github/workflows/build_arm_wheels.yml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_arm_wheels.yml b/.github/workflows/build_arm_wheels.yml index 1cff25ea235..020d618057f 100644 --- a/.github/workflows/build_arm_wheels.yml +++ b/.github/workflows/build_arm_wheels.yml @@ -133,7 +133,17 @@ jobs: env: TWINE_USERNAME: __token__ TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - + # #build for macos12 arm64(Apple Silicon) # build_wheels_macos_arm64: - + + runner_cleanup_arm64: + name: Wipe Caches on Self-Hosted Runners + needs: [ build_wheels_linux_arm64 ] + if: always() + runs-on: + - self-hosted + - ARM64 + steps: + - name: Actions Cache Cleanup + run: if [ "$(which action_cleanup)" != "" ]; then action_cleanup; else echo bypass; fi