Fixes building the vineyard_llm wheels on Linux (#1921)

Signed-off-by: Tao He <[email protected]>
v6d-io · Jun 20, 2024 · d2c86bb · d2c86bb
1 parent b5315cf
commit d2c86bb
Show file tree

Hide file tree

Showing 2 changed files with 20 additions and 12 deletions.
diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml
@@ -329,45 +329,45 @@ jobs:
         if: false
         uses: mxschmitt/action-tmate@v3
 
-      - name: Run llm tests
+      - name: Run cpp tests
         run: |
           export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib:/usr/local/lib64:/usr/local/lib/x86_64-linux-gnu
           export VINEYARD_DATA_DIR=`pwd`/gstest
           export TMPDIR="${TMPDIR:-$(dirname $(mktemp))}"
 
           rm -rf default.etcd
           rm -rf /dev/shm/etcd*
-          python3 test/runner.py $RUNNER_ARGS --with-llm
+          python3 test/runner.py $RUNNER_ARGS --with-cpp
 
-      - name: Run cpp tests
+      - name: Run python tests
         run: |
           export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib:/usr/local/lib64:/usr/local/lib/x86_64-linux-gnu
           export VINEYARD_DATA_DIR=`pwd`/gstest
           export TMPDIR="${TMPDIR:-$(dirname $(mktemp))}"
 
           rm -rf default.etcd
           rm -rf /dev/shm/etcd*
-          python3 test/runner.py $RUNNER_ARGS --with-cpp
+          python3 test/runner.py $RUNNER_ARGS --with-python
 
-      - name: Run python tests
+      - name: Run deployment tests
         run: |
           export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib:/usr/local/lib64:/usr/local/lib/x86_64-linux-gnu
           export VINEYARD_DATA_DIR=`pwd`/gstest
           export TMPDIR="${TMPDIR:-$(dirname $(mktemp))}"
 
           rm -rf default.etcd
           rm -rf /dev/shm/etcd*
-          python3 test/runner.py $RUNNER_ARGS --with-python
+          python3 test/runner.py $RUNNER_ARGS --with-deployment --with-migration
 
-      - name: Run deployment tests
+      - name: Run llm tests
         run: |
           export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib:/usr/local/lib64:/usr/local/lib/x86_64-linux-gnu
           export VINEYARD_DATA_DIR=`pwd`/gstest
           export TMPDIR="${TMPDIR:-$(dirname $(mktemp))}"
 
           rm -rf default.etcd
           rm -rf /dev/shm/etcd*
-          python3 test/runner.py $RUNNER_ARGS --with-deployment --with-migration
+          python3 test/runner.py $RUNNER_ARGS --with-llm
 
       - name: Run llm python tests
         run: |

diff --git a/docker/pypa/Dockerfile.manylinux1-wheel b/docker/pypa/Dockerfile.manylinux1-wheel
@@ -38,22 +38,30 @@ RUN cd /work/v6d && \
              -DBUILD_SHARED_LIBS=OFF \
              -DBUILD_VINEYARD_SERVER=OFF \
              -DBUILD_VINEYARD_CLIENT=ON \
-             -DBUILD_VINEYARD_PYTHON_BINDINGS=ON \
-             -DBUILD_VINEYARD_PYPI_PACKAGES=ON \
-             -DBUILD_VINEYARD_BASIC=OFF \
+             -DBUILD_VINEYARD_BASIC=ON \
              -DBUILD_VINEYARD_IO=OFF \
              -DBUILD_VINEYARD_GRAPH=OFF \
-             -DBUILD_VINEYARD_MIGRATION=ON \
              -DBUILD_VINEYARD_HOSSEINMOEIN_DATAFRAME=OFF \
+             -DBUILD_VINEYARD_LLM_CACHE=ON \
+             -DBUILD_VINEYARD_MIGRATION=ON \
+             -DBUILD_VINEYARD_PYTHON_BINDINGS=ON \
+             -DBUILD_VINEYARD_PYPI_PACKAGES=ON \
              -DBUILD_VINEYARD_TESTS=OFF \
              -DBUILD_VINEYARD_TESTS_ALL=OFF \
              -DBUILD_VINEYARD_COVERAGE=OFF \
              -DBUILD_VINEYARD_PROFILING=OFF \
              -DCMAKE_BUILD_TYPE=Release \
              -DPYTHON_EXECUTABLE=/opt/python/$python/bin/python && \
+    cd /work/v6d/build && \
+    rm -rf bdist.linux* lib.linux* && \
     make vineyard_client_python -j$(nproc) && \
     cd .. && \
     /opt/python/$python/bin/python setup.py bdist_wheel && \
+    cd /work/v6d/build && \
+    rm -rf bdist.linux* lib.linux* && \
+    make vineyard_llm_python -j$(nproc) && \
+    cd .. && \
+    /opt/python/$python/bin/python setup_llm.py bdist_wheel && \
     export LD_LIBRARY_PATH=/work/v6d/build/lib:$LD_LIBRARY_PATH && \
     for pylibs in /opt/_internal/tools/lib/*; do\
         sed -i 's/p.error/logger.warning/g' $pylibs/site-packages/auditwheel/main_repair.py || true; \