From 2888eac8efa92db9f3593cae9e6c88a61f062e63 Mon Sep 17 00:00:00 2001 From: Ruihang Lai Date: Mon, 18 Nov 2024 01:24:26 -0500 Subject: [PATCH] [Package] Build once for each Python version PyBind is Python-version-depended. Previously we only build XGrammar once (where PyBind is built only once as well) and then build the wheel for each Python version, which causes Python version error. This PR updates the build scripts to build XGrammar once for each Python version. --- .github/workflows/wheel_mac.yaml | 51 ++++++++++++++--------- .github/workflows/wheel_manylinux.yaml | 4 -- .github/workflows/wheel_windows.yaml | 51 ++++++++++++++--------- scripts/build_xgrammar_wheel_manylinux.sh | 38 ++++++++++------- 4 files changed, 88 insertions(+), 56 deletions(-) diff --git a/.github/workflows/wheel_mac.yaml b/.github/workflows/wheel_mac.yaml index 3f591b0..86af3db 100644 --- a/.github/workflows/wheel_mac.yaml +++ b/.github/workflows/wheel_mac.yaml @@ -32,70 +32,83 @@ jobs: - name: Sync XGrammar Package run: | python scripts/sync_package.py --package . --package-name xgrammar --version ${{ github.ref_name }} - # Use conda for LLVM dep - - uses: conda-incubator/setup-miniconda@v3 - with: - activate-environment: xgrammar-build - channel-priority: strict - environment-file: conda/build-environment.yaml - auto-activate-base: false - - name: Conda info - run: | - conda info - conda list - python --version - - name: Build@MacOS - run: >- - scripts/build_xgrammar_lib_osx.sh ${{ matrix.platform == 'macos-13' && '10.15' || '13.02' }} # Build wheel for different python versions - name: Setup@Py39 - uses: conda-incubator/setup-miniconda@v2 + uses: conda-incubator/setup-miniconda@v3 with: activate-environment: build-Py39 + channel-priority: strict + environment-file: conda/build-environment.yaml python-version: 3.9 auto-activate-base: false + - name: XGrammar-Build@Py39 + run: >- + scripts/build_xgrammar_lib_osx.sh ${{ matrix.platform == 'macos-13' && '10.15' || '13.02' }} - name: Wheel-Build@Py39 run: | python --version python -m pip install setuptools Cython wheel cd python python setup.py bdist_wheel + cd ../build + make clean - name: Setup@Py310 - uses: conda-incubator/setup-miniconda@v2 + uses: conda-incubator/setup-miniconda@v3 with: activate-environment: build-Py310 + channel-priority: strict + environment-file: conda/build-environment.yaml python-version: '3.10' auto-activate-base: false + - name: XGrammar-Build@Py310 + run: >- + scripts/build_xgrammar_lib_osx.sh ${{ matrix.platform == 'macos-13' && '10.15' || '13.02' }} - name: Wheel-Build@Py310 run: | python --version python -m pip install setuptools Cython wheel cd python python setup.py bdist_wheel + cd ../build + make clean - name: Setup@Py311 - uses: conda-incubator/setup-miniconda@v2 + uses: conda-incubator/setup-miniconda@v3 with: activate-environment: build-Py311 + channel-priority: strict + environment-file: conda/build-environment.yaml python-version: '3.11' auto-activate-base: false + - name: XGrammar-Build@Py311 + run: >- + scripts/build_xgrammar_lib_osx.sh ${{ matrix.platform == 'macos-13' && '10.15' || '13.02' }} - name: Wheel-Build@Py311 run: | python --version python -m pip install setuptools Cython wheel cd python python setup.py bdist_wheel + cd ../build + make clean - name: Setup@Py312 - uses: conda-incubator/setup-miniconda@v2 + uses: conda-incubator/setup-miniconda@v3 with: activate-environment: build-Py312 + channel-priority: strict + environment-file: conda/build-environment.yaml python-version: '3.12' auto-activate-base: false + - name: XGrammar-Build@Py312 + run: >- + scripts/build_xgrammar_lib_osx.sh ${{ matrix.platform == 'macos-13' && '10.15' || '13.02' }} - name: Wheel-Build@Py312 run: | python --version python -m pip install setuptools Cython wheel cd python python setup.py bdist_wheel + cd ../build + make clean - uses: actions/upload-artifact@v4 with: name: xgrammar-wheel-${{ github.ref_name }}-${{ matrix.platform }} diff --git a/.github/workflows/wheel_manylinux.yaml b/.github/workflows/wheel_manylinux.yaml index f8e5bc5..8b48b0b 100644 --- a/.github/workflows/wheel_manylinux.yaml +++ b/.github/workflows/wheel_manylinux.yaml @@ -35,10 +35,6 @@ jobs: - name: Checkout source run: | git clone https://github.com/mlc-ai/package package --recursive - - name: Setup script env - run: | - rm -rf conda - ln -s package/3rdparty/tlcpack/conda conda - name: Sync XGrammar Package run: | python scripts/sync_package.py --package . --package-name xgrammar --version ${{ github.ref_name }} diff --git a/.github/workflows/wheel_windows.yaml b/.github/workflows/wheel_windows.yaml index 3b86a16..473a355 100644 --- a/.github/workflows/wheel_windows.yaml +++ b/.github/workflows/wheel_windows.yaml @@ -31,70 +31,83 @@ jobs: - name: Sync XGrammar Package run: | python scripts/sync_package.py --package . --package-name xgrammar --version ${{ github.ref_name }} - # Use conda for LLVM dep - - uses: conda-incubator/setup-miniconda@v3 - with: - activate-environment: xgrammar-build - channel-priority: strict - environment-file: conda/build-environment.yaml - auto-activate-base: false - - name: Conda info - run: | - conda info - conda list - python --version - - name: Build@Win - run: >- - scripts/build_xgrammar_lib_win.bat # Build wheel for different python versions - name: Setup@Py39 - uses: conda-incubator/setup-miniconda@v2 + uses: conda-incubator/setup-miniconda@v3 with: activate-environment: build-Py39 + channel-priority: strict + environment-file: conda/build-environment.yaml python-version: 3.9 auto-activate-base: false + - name: XGrammar-Build@Py39 + run: >- + scripts/build_xgrammar_lib_win.bat - name: Wheel-Build@Py39 run: | python --version python -m pip install setuptools Cython wheel cd python python setup.py bdist_wheel + cd ../build + make clean - name: Setup@Py310 - uses: conda-incubator/setup-miniconda@v2 + uses: conda-incubator/setup-miniconda@v3 with: activate-environment: build-Py310 + channel-priority: strict + environment-file: conda/build-environment.yaml python-version: '3.10' auto-activate-base: false + - name: XGrammar-Build@Py310 + run: >- + scripts/build_xgrammar_lib_win.bat - name: Wheel-Build@Py310 run: | python --version python -m pip install setuptools Cython wheel cd python python setup.py bdist_wheel + cd ../build + make clean - name: Setup@Py311 - uses: conda-incubator/setup-miniconda@v2 + uses: conda-incubator/setup-miniconda@v3 with: activate-environment: build-Py311 + channel-priority: strict + environment-file: conda/build-environment.yaml python-version: '3.11' auto-activate-base: false + - name: XGrammar-Build@Py311 + run: >- + scripts/build_xgrammar_lib_win.bat - name: Wheel-Build@Py311 run: | python --version python -m pip install setuptools Cython wheel cd python python setup.py bdist_wheel + cd ../build + make clean - name: Setup@Py312 - uses: conda-incubator/setup-miniconda@v2 + uses: conda-incubator/setup-miniconda@v3 with: activate-environment: build-Py312 + channel-priority: strict + environment-file: conda/build-environment.yaml python-version: '3.12' auto-activate-base: false + - name: XGrammar-Build@Py312 + run: >- + scripts/build_xgrammar_lib_win.bat - name: Wheel-Build@Py312 run: | python --version python -m pip install setuptools Cython wheel cd python python setup.py bdist_wheel + cd ../build + make clean - uses: actions/upload-artifact@v4 with: name: xgrammar-wheel-${{ github.ref_name }}-windows diff --git a/scripts/build_xgrammar_wheel_manylinux.sh b/scripts/build_xgrammar_wheel_manylinux.sh index 55513f0..55b4a03 100755 --- a/scripts/build_xgrammar_wheel_manylinux.sh +++ b/scripts/build_xgrammar_wheel_manylinux.sh @@ -38,6 +38,7 @@ function audit_xgrammar_wheel() { ${XGRAMMAR_PYTHON_DIR}/*.egg-info } +XGRAMMAR_SOURCE_DIR="/workspace" XGRAMMAR_PYTHON_DIR="/workspace/python" PYTHON_VERSIONS_CPU=("3.9" "3.10" "3.11" "3.12") @@ -66,30 +67,36 @@ AUDITWHEEL_OPTS="--exclude libtorch --exclude libtorch_cpu --exclude libtorch_py # config the cmake cd /workspace -# setup config.cmake -echo set\(XGRAMMAR_BUILD_PYTHON_BINDINGS ON\) >>config.cmake -echo set\(XGRAMMAR_BUILD_CXX_TESTS OFF\) >>config.cmake - -# compile the xgrammar -python3 -m pip install pybind11 -python3 -m pip install torch --index-url https://download.pytorch.org/whl/cpu -mkdir -p build -cd build -cmake .. -make -j4 -find . -type d -name 'CMakeFiles' -exec rm -rf {} + - UNICODE_WIDTH=32 # Dummy value, irrelevant for Python 3 +source /opt/conda/etc/profile.d/conda.sh + # Not all manylinux Docker images will have all Python versions, # so check the existing python versions before generating packages for python_version in ${PYTHON_VERSIONS[*]}; do echo "> Looking for Python ${python_version}." - # Remove the . in version string, e.g. "3.8" turns into "38" python_version_str="$(echo "${python_version}" | sed -r 's/\.//g')" cpython_dir="/opt/conda/envs/py${python_version_str}/" + # compile xgrammar on the particular python version + conda activate py${python_version_str} + + # setup config.cmake + cd $XGRAMMAR_SOURCE_DIR + rm -rf config.cmake + echo set\(XGRAMMAR_BUILD_PYTHON_BINDINGS ON\) >>config.cmake + echo set\(XGRAMMAR_BUILD_CXX_TESTS OFF\) >>config.cmake + + python3 -m pip install pybind11 + python3 -m pip install torch --index-url https://download.pytorch.org/whl/cpu + rm -rf build + mkdir -p build + cd build + cmake .. + make -j4 + find . -type d -name 'CMakeFiles' -exec rm -rf {} + + # For compatibility in environments where Conda is not installed, # revert back to previous method of locating cpython_dir. if ! [ -d "${cpython_dir}" ]; then @@ -106,4 +113,7 @@ for python_version in ${PYTHON_VERSIONS[*]}; do echo "Python ${python_version} not found. Skipping." fi + cd $XGRAMMAR_SOURCE_DIR/build + make clean + done