diff --git a/.github/workflows/build_arm_wheels.yml b/.github/workflows/build_arm_wheels.yml new file mode 100644 index 00000000000..020d618057f --- /dev/null +++ b/.github/workflows/build_arm_wheels.yml @@ -0,0 +1,149 @@ +name: Build arm64 + +on: + # for test action + pull_request: + branches: + - main + - feat/add-arm-wheel-build + # end for test action + workflow_dispatch: + inputs: + TAG_NAME: + description: 'Release Version Tag' + required: true + release: + types: [created] + +defaults: + run: + shell: bash -leo pipefail {0} + +jobs: + build_wheels_linux_arm64: + name: ${{ matrix.os }} py${{ matrix.python-version }} + runs-on: + - self-hosted + - ARM64 + strategy: + fail-fast: false + matrix: + os: [ ubuntu-20.04 ] + env: + RUNNER_OS: ${{ matrix.os }} + PYTHON_VERSIONS: "3.8 3.9 3.10 3.11" + steps: + - name: Install clang++ for Ubuntu + if: matrix.os == 'ubuntu-20.04' + run: | + pwd + uname -a + wget https://apt.llvm.org/llvm.sh + chmod +x llvm.sh + sudo ./llvm.sh 15 + which clang++-15 + clang++-15 --version + sudo apt-get install -y make cmake ccache ninja-build yasm gawk wget + ccache -s + - name: Update git + run: | + sudo add-apt-repository ppa:git-core/ppa -y + sudo apt-get update + sudo apt-get install -y git + git --version + shell: bash -l {0} + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Restore submodules cache + uses: actions/cache/restore@v3 + id: cache + with: + path: | + contrib/** + key: | + submodule-${{ hashFiles('.gitmodules') }} + - name: Update submodules if cache miss + if: steps.cache.outputs.cache-hit != 'true' + run: | + git submodule update --init --recursive --jobs 4 + - name: Save submodules cache + if: steps.cache.outputs.cache-hit != 'true' + uses: actions/cache/save@v3 + with: + path: | + contrib/** + key: | + submodule-${{ hashFiles('.gitmodules') }} + - name: ccache + uses: hendrikmuhs/ccache-action@v1.2 + with: + key: ${{ matrix.os }} + max-size: 5G + append-timestamp: true + - name: remove old clang and link clang-15 to clang + if: matrix.os == 'ubuntu-20.04' + run: | + sudo rm /usr/bin/clang + sudo ln -s /usr/bin/clang-15 /usr/bin/clang + sudo rm /usr/bin/clang++ + sudo ln -s /usr/bin/clang++-15 /usr/bin/clang++ + which clang++ + clang++ --version + - name: Make linux-arm64 + run: | + #make linux-arm64 + bash -l -e chdb/build_linux_arm64.sh + continue-on-error: false + - name: Check ccache statistics + run: | + ccache -s + ls -lh chdb + df -h + - name: Install patchelf from github + run: | + pyenv local "${{ matrix.python-version }}" + wget https://github.com/NixOS/patchelf/releases/download/0.18.0/patchelf-0.18.0-aarch64.tar.gz -O patchelf.tar.gz + tar -xvf patchelf.tar.gz + sudo cp bin/patchelf /usr/bin/ + sudo chmod +x /usr/bin/patchelf + patchelf --version + - name: Audit wheels + run: | + pyenv local "${{ matrix.python-version }}" + python3 -m pip install auditwheel + auditwheel -v repair -w dist/ --plat manylinux_2_17_aarch64 dist/*.whl + continue-on-error: false + - name: Show files + run: | + # e.g: remove chdb-0.11.4-cp310-cp310-linux_aarch64.whl, keep chdb-0.11.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl + sudo rm -f dist/*linux_aarch64.whl + ls -lh dist + shell: bash + continue-on-error: false + - uses: actions/upload-artifact@v3 + with: + path: ./dist/*.whl + - name: Upload pypi + if: startsWith(github.ref, 'refs/tags/v') + run: | + pyenv local "${{ matrix.python-version }}" + python3 -m pip install twine + python3 -m twine upload wheelhouse/*.whl + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + + # #build for macos12 arm64(Apple Silicon) + # build_wheels_macos_arm64: + + runner_cleanup_arm64: + name: Wipe Caches on Self-Hosted Runners + needs: [ build_wheels_linux_arm64 ] + if: always() + runs-on: + - self-hosted + - ARM64 + steps: + - name: Actions Cache Cleanup + run: if [ "$(which action_cleanup)" != "" ]; then action_cleanup; else echo bypass; fi diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index f4a1e51a776..3bd49b0a588 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -1,21 +1,13 @@ -name: Build +name: Build X86 on: - push: - tags: - - 'v*' - branches: - - main - paths-ignore: - - '**.md' - - '**.yml' - - '.github/**' - - 'examples/**' - - 'tests/**' - pull_request: - branches: - - main workflow_dispatch: + inputs: + TAG_NAME: + description: 'Release Version Tag' + required: true + release: + types: [created] jobs: build_wheels_linux: diff --git a/Makefile b/Makefile index f300ca810aa..bcc80cf3429 100644 --- a/Makefile +++ b/Makefile @@ -26,7 +26,12 @@ clean: mac-arm64: @echo "Make macOS arm64 whl" - packages/build_mac_arm64.sh + chdb/build_mac_arm64.sh + @echo "Done." + +linux-arm64: + @echo "Make linux arm64 whl" + chdb/build_linux_arm64.sh @echo "Done." build: clean buildlib wheel diff --git a/chdb/build_linux_arm64.sh b/chdb/build_linux_arm64.sh new file mode 100755 index 00000000000..410249ceb81 --- /dev/null +++ b/chdb/build_linux_arm64.sh @@ -0,0 +1,58 @@ +#!/bin/bash -e +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +PROJ_DIR=$(dirname ${DIR}) + +[ -z "${PYTHON_VERSIONS}" ] && { echo "please provide PYTHON_VERSIONS env, e.g: PYTHON_VERSIONS='3.8 3.9'"; exit 1; } + +# echo ${PROJ_DIR} + +for PY_VER in ${PYTHON_VERSIONS}; do + cd ${PROJ_DIR} + pyenv local "${PY_VER}" + python3 --version + python3 -m pip install pybind11 + export CC=/usr/bin/clang + export CXX=/usr/bin/clang++ + # Install universal2 pkg + python3 -VV + python3-config --includes + # if python3 -VV does not contain ${PY_VER}, then exit + if ! python3 -VV 2>&1 | grep -q "${PY_VER}"; then + echo "Error: Required version of Python (${PY_VER}) not found. Aborting." + exit 1 + fi + + python3 -m pip install -U pybind11 wheel build tox + rm -rf ./buildlib + + ./chdb/build.sh + cd chdb && python3 -c "import _chdb; res = _chdb.query('select 1112222222,555', 'JSON'); print(res)" && cd - + + ./gen_manifest.sh + cat ./MANIFEST.in + + # try delete + whl_file=$(find dist | grep 'whl$' | grep cp${PY_VER//./}-cp${PY_VER//./} || echo "notfound.whl") + rm -f ${whl_file} || : + + python3 -m build --wheel + + python3 -m wheel tags --platform-tag=manylinux_2_17_aarch64 + + find dist + + python3 -m pip install pandas pyarrow psutil + find dist + whl_file=$(find dist | grep 'whl$' | grep cp${PY_VER//./}-cp${PY_VER//./}) + python3 -m pip install --force-reinstall ${whl_file} + + python3 -c "import chdb; res = chdb.query('select version()', 'CSV'); print(res)" + + python3 -m chdb "SELECT 1, 'ab'" arrowtable + + python3 -m chdb "SELECT 1, 'ab'" dataframe + + bash -x ./chdb/test_smoke.sh + + make test +done diff --git a/packages/build_mac_arm64.sh b/chdb/build_mac_arm64.sh similarity index 100% rename from packages/build_mac_arm64.sh rename to chdb/build_mac_arm64.sh