From d2c86bb70e8e7cddd8bfabbdfe874b22e3eaf92f Mon Sep 17 00:00:00 2001 From: Tao He Date: Thu, 20 Jun 2024 12:14:10 +0800 Subject: [PATCH] Fixes building the vineyard_llm wheels on Linux (#1921) Signed-off-by: Tao He --- .github/workflows/build-test.yml | 16 ++++++++-------- docker/pypa/Dockerfile.manylinux1-wheel | 16 ++++++++++++---- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index edbd8e2e1..dbc807053 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -329,7 +329,7 @@ jobs: if: false uses: mxschmitt/action-tmate@v3 - - name: Run llm tests + - name: Run cpp tests run: | export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib:/usr/local/lib64:/usr/local/lib/x86_64-linux-gnu export VINEYARD_DATA_DIR=`pwd`/gstest @@ -337,9 +337,9 @@ jobs: rm -rf default.etcd rm -rf /dev/shm/etcd* - python3 test/runner.py $RUNNER_ARGS --with-llm + python3 test/runner.py $RUNNER_ARGS --with-cpp - - name: Run cpp tests + - name: Run python tests run: | export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib:/usr/local/lib64:/usr/local/lib/x86_64-linux-gnu export VINEYARD_DATA_DIR=`pwd`/gstest @@ -347,9 +347,9 @@ jobs: rm -rf default.etcd rm -rf /dev/shm/etcd* - python3 test/runner.py $RUNNER_ARGS --with-cpp + python3 test/runner.py $RUNNER_ARGS --with-python - - name: Run python tests + - name: Run deployment tests run: | export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib:/usr/local/lib64:/usr/local/lib/x86_64-linux-gnu export VINEYARD_DATA_DIR=`pwd`/gstest @@ -357,9 +357,9 @@ jobs: rm -rf default.etcd rm -rf /dev/shm/etcd* - python3 test/runner.py $RUNNER_ARGS --with-python + python3 test/runner.py $RUNNER_ARGS --with-deployment --with-migration - - name: Run deployment tests + - name: Run llm tests run: | export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib:/usr/local/lib64:/usr/local/lib/x86_64-linux-gnu export VINEYARD_DATA_DIR=`pwd`/gstest @@ -367,7 +367,7 @@ jobs: rm -rf default.etcd rm -rf /dev/shm/etcd* - python3 test/runner.py $RUNNER_ARGS --with-deployment --with-migration + python3 test/runner.py $RUNNER_ARGS --with-llm - name: Run llm python tests run: | diff --git a/docker/pypa/Dockerfile.manylinux1-wheel b/docker/pypa/Dockerfile.manylinux1-wheel index e33abdc58..bf0096939 100644 --- a/docker/pypa/Dockerfile.manylinux1-wheel +++ b/docker/pypa/Dockerfile.manylinux1-wheel @@ -38,22 +38,30 @@ RUN cd /work/v6d && \ -DBUILD_SHARED_LIBS=OFF \ -DBUILD_VINEYARD_SERVER=OFF \ -DBUILD_VINEYARD_CLIENT=ON \ - -DBUILD_VINEYARD_PYTHON_BINDINGS=ON \ - -DBUILD_VINEYARD_PYPI_PACKAGES=ON \ - -DBUILD_VINEYARD_BASIC=OFF \ + -DBUILD_VINEYARD_BASIC=ON \ -DBUILD_VINEYARD_IO=OFF \ -DBUILD_VINEYARD_GRAPH=OFF \ - -DBUILD_VINEYARD_MIGRATION=ON \ -DBUILD_VINEYARD_HOSSEINMOEIN_DATAFRAME=OFF \ + -DBUILD_VINEYARD_LLM_CACHE=ON \ + -DBUILD_VINEYARD_MIGRATION=ON \ + -DBUILD_VINEYARD_PYTHON_BINDINGS=ON \ + -DBUILD_VINEYARD_PYPI_PACKAGES=ON \ -DBUILD_VINEYARD_TESTS=OFF \ -DBUILD_VINEYARD_TESTS_ALL=OFF \ -DBUILD_VINEYARD_COVERAGE=OFF \ -DBUILD_VINEYARD_PROFILING=OFF \ -DCMAKE_BUILD_TYPE=Release \ -DPYTHON_EXECUTABLE=/opt/python/$python/bin/python && \ + cd /work/v6d/build && \ + rm -rf bdist.linux* lib.linux* && \ make vineyard_client_python -j$(nproc) && \ cd .. && \ /opt/python/$python/bin/python setup.py bdist_wheel && \ + cd /work/v6d/build && \ + rm -rf bdist.linux* lib.linux* && \ + make vineyard_llm_python -j$(nproc) && \ + cd .. && \ + /opt/python/$python/bin/python setup_llm.py bdist_wheel && \ export LD_LIBRARY_PATH=/work/v6d/build/lib:$LD_LIBRARY_PATH && \ for pylibs in /opt/_internal/tools/lib/*; do\ sed -i 's/p.error/logger.warning/g' $pylibs/site-packages/auditwheel/main_repair.py || true; \