Skip to content

Commit

Permalink
Add arm64 linux build (#70)
Browse files Browse the repository at this point in the history
* Add arm build

* Update build_arm_whells.yml

* fix build wheel

* Cleanup Job

Cleanup Job for Self-Hosted runners to reclaim disk space after successful or failed builds.

---------

Co-authored-by: Lorenzo Mangani <[email protected]>
  • Loading branch information
2 people authored and auxten committed Nov 9, 2023
1 parent 28d388e commit 85ed59e
Show file tree
Hide file tree
Showing 5 changed files with 220 additions and 16 deletions.
149 changes: 149 additions & 0 deletions .github/workflows/build_arm_wheels.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,149 @@
name: Build arm64

on:
# for test action
pull_request:
branches:
- main
- feat/add-arm-wheel-build
# end for test action
workflow_dispatch:
inputs:
TAG_NAME:
description: 'Release Version Tag'
required: true
release:
types: [created]

defaults:
run:
shell: bash -leo pipefail {0}

jobs:
build_wheels_linux_arm64:
name: ${{ matrix.os }} py${{ matrix.python-version }}
runs-on:
- self-hosted
- ARM64
strategy:
fail-fast: false
matrix:
os: [ ubuntu-20.04 ]
env:
RUNNER_OS: ${{ matrix.os }}
PYTHON_VERSIONS: "3.8 3.9 3.10 3.11"
steps:
- name: Install clang++ for Ubuntu
if: matrix.os == 'ubuntu-20.04'
run: |
pwd
uname -a
wget https://apt.llvm.org/llvm.sh
chmod +x llvm.sh
sudo ./llvm.sh 15
which clang++-15
clang++-15 --version
sudo apt-get install -y make cmake ccache ninja-build yasm gawk wget
ccache -s
- name: Update git
run: |
sudo add-apt-repository ppa:git-core/ppa -y
sudo apt-get update
sudo apt-get install -y git
git --version
shell: bash -l {0}
- uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Restore submodules cache
uses: actions/cache/restore@v3
id: cache
with:
path: |
contrib/**
key: |
submodule-${{ hashFiles('.gitmodules') }}
- name: Update submodules if cache miss
if: steps.cache.outputs.cache-hit != 'true'
run: |
git submodule update --init --recursive --jobs 4
- name: Save submodules cache
if: steps.cache.outputs.cache-hit != 'true'
uses: actions/cache/save@v3
with:
path: |
contrib/**
key: |
submodule-${{ hashFiles('.gitmodules') }}
- name: ccache
uses: hendrikmuhs/[email protected]
with:
key: ${{ matrix.os }}
max-size: 5G
append-timestamp: true
- name: remove old clang and link clang-15 to clang
if: matrix.os == 'ubuntu-20.04'
run: |
sudo rm /usr/bin/clang
sudo ln -s /usr/bin/clang-15 /usr/bin/clang
sudo rm /usr/bin/clang++
sudo ln -s /usr/bin/clang++-15 /usr/bin/clang++
which clang++
clang++ --version
- name: Make linux-arm64
run: |
#make linux-arm64
bash -l -e chdb/build_linux_arm64.sh
continue-on-error: false
- name: Check ccache statistics
run: |
ccache -s
ls -lh chdb
df -h
- name: Install patchelf from github
run: |
pyenv local "${{ matrix.python-version }}"
wget https://github.com/NixOS/patchelf/releases/download/0.18.0/patchelf-0.18.0-aarch64.tar.gz -O patchelf.tar.gz
tar -xvf patchelf.tar.gz
sudo cp bin/patchelf /usr/bin/
sudo chmod +x /usr/bin/patchelf
patchelf --version
- name: Audit wheels
run: |
pyenv local "${{ matrix.python-version }}"
python3 -m pip install auditwheel
auditwheel -v repair -w dist/ --plat manylinux_2_17_aarch64 dist/*.whl
continue-on-error: false
- name: Show files
run: |
# e.g: remove chdb-0.11.4-cp310-cp310-linux_aarch64.whl, keep chdb-0.11.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl
sudo rm -f dist/*linux_aarch64.whl
ls -lh dist
shell: bash
continue-on-error: false
- uses: actions/upload-artifact@v3
with:
path: ./dist/*.whl
- name: Upload pypi
if: startsWith(github.ref, 'refs/tags/v')
run: |
pyenv local "${{ matrix.python-version }}"
python3 -m pip install twine
python3 -m twine upload wheelhouse/*.whl
env:
TWINE_USERNAME: __token__
TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}

# #build for macos12 arm64(Apple Silicon)
# build_wheels_macos_arm64:

runner_cleanup_arm64:
name: Wipe Caches on Self-Hosted Runners
needs: [ build_wheels_linux_arm64 ]
if: always()
runs-on:
- self-hosted
- ARM64
steps:
- name: Actions Cache Cleanup
run: if [ "$(which action_cleanup)" != "" ]; then action_cleanup; else echo bypass; fi
22 changes: 7 additions & 15 deletions .github/workflows/build_wheels.yml
Original file line number Diff line number Diff line change
@@ -1,21 +1,13 @@
name: Build
name: Build X86

on:
push:
tags:
- 'v*'
branches:
- main
paths-ignore:
- '**.md'
- '**.yml'
- '.github/**'
- 'examples/**'
- 'tests/**'
pull_request:
branches:
- main
workflow_dispatch:
inputs:
TAG_NAME:
description: 'Release Version Tag'
required: true
release:
types: [created]

jobs:
build_wheels_linux:
Expand Down
7 changes: 6 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,12 @@ clean:

mac-arm64:
@echo "Make macOS arm64 whl"
packages/build_mac_arm64.sh
chdb/build_mac_arm64.sh
@echo "Done."

linux-arm64:
@echo "Make linux arm64 whl"
chdb/build_linux_arm64.sh
@echo "Done."

build: clean buildlib wheel
58 changes: 58 additions & 0 deletions chdb/build_linux_arm64.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
#!/bin/bash -e
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
PROJ_DIR=$(dirname ${DIR})

[ -z "${PYTHON_VERSIONS}" ] && { echo "please provide PYTHON_VERSIONS env, e.g: PYTHON_VERSIONS='3.8 3.9'"; exit 1; }

# echo ${PROJ_DIR}

for PY_VER in ${PYTHON_VERSIONS}; do
cd ${PROJ_DIR}
pyenv local "${PY_VER}"
python3 --version
python3 -m pip install pybind11
export CC=/usr/bin/clang
export CXX=/usr/bin/clang++
# Install universal2 pkg
python3 -VV
python3-config --includes
# if python3 -VV does not contain ${PY_VER}, then exit
if ! python3 -VV 2>&1 | grep -q "${PY_VER}"; then
echo "Error: Required version of Python (${PY_VER}) not found. Aborting."
exit 1
fi

python3 -m pip install -U pybind11 wheel build tox
rm -rf ./buildlib

./chdb/build.sh
cd chdb && python3 -c "import _chdb; res = _chdb.query('select 1112222222,555', 'JSON'); print(res)" && cd -

./gen_manifest.sh
cat ./MANIFEST.in

# try delete
whl_file=$(find dist | grep 'whl$' | grep cp${PY_VER//./}-cp${PY_VER//./} || echo "notfound.whl")
rm -f ${whl_file} || :

python3 -m build --wheel

python3 -m wheel tags --platform-tag=manylinux_2_17_aarch64

find dist

python3 -m pip install pandas pyarrow psutil
find dist
whl_file=$(find dist | grep 'whl$' | grep cp${PY_VER//./}-cp${PY_VER//./})
python3 -m pip install --force-reinstall ${whl_file}

python3 -c "import chdb; res = chdb.query('select version()', 'CSV'); print(res)"

python3 -m chdb "SELECT 1, 'ab'" arrowtable

python3 -m chdb "SELECT 1, 'ab'" dataframe

bash -x ./chdb/test_smoke.sh

make test
done
File renamed without changes.

0 comments on commit 85ed59e

Please sign in to comment.