[Package] Build once for each Python version

PyBind is Python-version-depended. Previously we only build XGrammar once (where PyBind is built only once as well) and then build the wheel for each Python version, which causes Python version error. This PR updates the build scripts to build XGrammar once for each Python version.
mlc-ai · Nov 18, 2024 · 8c37117 · 8c37117
1 parent 642a209
commit 8c37117
Show file tree

Hide file tree

Showing 4 changed files with 77 additions and 59 deletions.
diff --git a/.github/workflows/wheel_mac.yaml b/.github/workflows/wheel_mac.yaml
@@ -32,70 +32,79 @@ jobs:
     - name: Sync XGrammar Package
       run: |
         python scripts/sync_package.py --package . --package-name xgrammar --version ${{ github.ref_name }}
-    # Use conda for LLVM dep
-    - uses: conda-incubator/setup-miniconda@v3
-      with:
-        activate-environment: xgrammar-build
-        channel-priority: strict
-        environment-file: conda/build-environment.yaml
-        auto-activate-base: false
-    - name: Conda info
-      run: |
-        conda info
-        conda list
-        python --version
-    - name: Build@MacOS
-      run: >-
-        scripts/build_xgrammar_lib_osx.sh ${{ matrix.platform == 'macos-13' && '10.15' || '13.02' }}
     # Build wheel for different python versions
     - name: Setup@Py39
-      uses: conda-incubator/setup-miniconda@v2
+      uses: conda-incubator/setup-miniconda@v3
       with:
         activate-environment: build-Py39
+        channel-priority: strict
+        environment-file: conda/build-environment.yaml
         python-version: 3.9
         auto-activate-base: false
+    - name: XGrammar-Build@Py39
+      run: >-
+        scripts/build_xgrammar_lib_osx.sh ${{ matrix.platform == 'macos-13' && '10.15' || '13.02' }}
     - name: Wheel-Build@Py39
       run: |
         python --version
         python -m pip install setuptools Cython wheel
         cd python
         python setup.py bdist_wheel
+        rm -rf xgrammar/*.so
     - name: Setup@Py310
-      uses: conda-incubator/setup-miniconda@v2
+      uses: conda-incubator/setup-miniconda@v3
       with:
         activate-environment: build-Py310
+        channel-priority: strict
+        environment-file: conda/build-environment.yaml
         python-version: '3.10'
         auto-activate-base: false
+    - name: XGrammar-Build@Py310
+      run: >-
+        scripts/build_xgrammar_lib_osx.sh ${{ matrix.platform == 'macos-13' && '10.15' || '13.02' }}
     - name: Wheel-Build@Py310
       run: |
         python --version
         python -m pip install setuptools Cython wheel
         cd python
         python setup.py bdist_wheel
+        rm -rf xgrammar/*.so
     - name: Setup@Py311
-      uses: conda-incubator/setup-miniconda@v2
+      uses: conda-incubator/setup-miniconda@v3
       with:
         activate-environment: build-Py311
+        channel-priority: strict
+        environment-file: conda/build-environment.yaml
         python-version: '3.11'
         auto-activate-base: false
+    - name: XGrammar-Build@Py311
+      run: >-
+        scripts/build_xgrammar_lib_osx.sh ${{ matrix.platform == 'macos-13' && '10.15' || '13.02' }}
     - name: Wheel-Build@Py311
       run: |
         python --version
         python -m pip install setuptools Cython wheel
         cd python
         python setup.py bdist_wheel
+        rm -rf xgrammar/*.so
     - name: Setup@Py312
-      uses: conda-incubator/setup-miniconda@v2
+      uses: conda-incubator/setup-miniconda@v3
       with:
         activate-environment: build-Py312
+        channel-priority: strict
+        environment-file: conda/build-environment.yaml
         python-version: '3.12'
         auto-activate-base: false
+    - name: XGrammar-Build@Py312
+      run: >-
+        scripts/build_xgrammar_lib_osx.sh ${{ matrix.platform == 'macos-13' && '10.15' || '13.02' }}
     - name: Wheel-Build@Py312
       run: |
         python --version
         python -m pip install setuptools Cython wheel
         cd python
         python setup.py bdist_wheel
+        rm -rf xgrammar/*.so
     - uses: actions/upload-artifact@v4
       with:
         name: xgrammar-wheel-${{ github.ref_name }}-${{ matrix.platform }}

diff --git a/.github/workflows/wheel_manylinux.yaml b/.github/workflows/wheel_manylinux.yaml
@@ -35,10 +35,6 @@ jobs:
     - name: Checkout source
       run: |
         git clone https://github.com/mlc-ai/package package --recursive
-    - name: Setup script env
-      run: |
-        rm -rf conda
-        ln -s package/3rdparty/tlcpack/conda conda
     - name: Sync XGrammar Package
       run: |
         python scripts/sync_package.py --package . --package-name xgrammar --version ${{ github.ref_name }}

diff --git a/.github/workflows/wheel_windows.yaml b/.github/workflows/wheel_windows.yaml
@@ -31,70 +31,79 @@ jobs:
     - name: Sync XGrammar Package
       run: |
         python scripts/sync_package.py --package . --package-name xgrammar --version ${{ github.ref_name }}
-    # Use conda for LLVM dep
-    - uses: conda-incubator/setup-miniconda@v3
-      with:
-        activate-environment: xgrammar-build
-        channel-priority: strict
-        environment-file: conda/build-environment.yaml
-        auto-activate-base: false
-    - name: Conda info
-      run: |
-        conda info
-        conda list
-        python --version
-    - name: Build@Win
-      run: >-
-        scripts/build_xgrammar_lib_win.bat
     # Build wheel for different python versions
     - name: Setup@Py39
-      uses: conda-incubator/setup-miniconda@v2
+      uses: conda-incubator/setup-miniconda@v3
       with:
         activate-environment: build-Py39
+        channel-priority: strict
+        environment-file: conda/build-environment.yaml
         python-version: 3.9
         auto-activate-base: false
+    - name: XGrammar-Build@Py39
+      run: >-
+        scripts/build_xgrammar_lib_win.bat
     - name: Wheel-Build@Py39
       run: |
         python --version
         python -m pip install setuptools Cython wheel
         cd python
         python setup.py bdist_wheel
+        del /f xgrammar/*.pyd
     - name: Setup@Py310
-      uses: conda-incubator/setup-miniconda@v2
+      uses: conda-incubator/setup-miniconda@v3
       with:
         activate-environment: build-Py310
+        channel-priority: strict
+        environment-file: conda/build-environment.yaml
         python-version: '3.10'
         auto-activate-base: false
+    - name: XGrammar-Build@Py310
+      run: >-
+        scripts/build_xgrammar_lib_win.bat
     - name: Wheel-Build@Py310
       run: |
         python --version
         python -m pip install setuptools Cython wheel
         cd python
         python setup.py bdist_wheel
+        del /f xgrammar/*.pyd
     - name: Setup@Py311
-      uses: conda-incubator/setup-miniconda@v2
+      uses: conda-incubator/setup-miniconda@v3
       with:
         activate-environment: build-Py311
+        channel-priority: strict
+        environment-file: conda/build-environment.yaml
         python-version: '3.11'
         auto-activate-base: false
+    - name: XGrammar-Build@Py311
+      run: >-
+        scripts/build_xgrammar_lib_win.bat
     - name: Wheel-Build@Py311
       run: |
         python --version
         python -m pip install setuptools Cython wheel
         cd python
         python setup.py bdist_wheel
+        del /f xgrammar/*.pyd
     - name: Setup@Py312
-      uses: conda-incubator/setup-miniconda@v2
+      uses: conda-incubator/setup-miniconda@v3
       with:
         activate-environment: build-Py312
+        channel-priority: strict
+        environment-file: conda/build-environment.yaml
         python-version: '3.12'
         auto-activate-base: false
+    - name: XGrammar-Build@Py312
+      run: >-
+        scripts/build_xgrammar_lib_win.bat
     - name: Wheel-Build@Py312
       run: |
         python --version
         python -m pip install setuptools Cython wheel
         cd python
         python setup.py bdist_wheel
+        del /f xgrammar/*.pyd
     - uses: actions/upload-artifact@v4
       with:
         name: xgrammar-wheel-${{ github.ref_name }}-windows

diff --git a/scripts/build_xgrammar_wheel_manylinux.sh b/scripts/build_xgrammar_wheel_manylinux.sh
@@ -63,33 +63,37 @@ PYTHON_VERSIONS=${PYTHON_VERSIONS_CPU[*]}
 AUDITWHEEL_OPTS="--plat ${AUDITWHEEL_PLAT} -w repaired_wheels/"
 AUDITWHEEL_OPTS="--exclude libtorch --exclude libtorch_cpu --exclude libtorch_python ${AUDITWHEEL_OPTS}"
 
-# config the cmake
-cd /workspace
-
-# setup config.cmake
-echo set\(XGRAMMAR_BUILD_PYTHON_BINDINGS ON\) >>config.cmake
-echo set\(XGRAMMAR_BUILD_CXX_TESTS OFF\) >>config.cmake
-
-# compile the xgrammar
-python3 -m pip install pybind11
-python3 -m pip install torch --index-url https://download.pytorch.org/whl/cpu
-mkdir -p build
-cd build
-cmake ..
-make -j4
-find . -type d -name 'CMakeFiles' -exec rm -rf {} +
-
 UNICODE_WIDTH=32 # Dummy value, irrelevant for Python 3
 
+source /opt/conda/etc/profile.d/conda.sh
+
 # Not all manylinux Docker images will have all Python versions,
 # so check the existing python versions before generating packages
 for python_version in ${PYTHON_VERSIONS[*]}; do
 	echo "> Looking for Python ${python_version}."
-
 	# Remove the . in version string, e.g. "3.8" turns into "38"
 	python_version_str="$(echo "${python_version}" | sed -r 's/\.//g')"
 	cpython_dir="/opt/conda/envs/py${python_version_str}/"
 
+	# compile xgrammar on the particular python version
+	conda activate py${python_version_str}
+
+	# setup config.cmake
+	cd /workspace
+	rm -rf ${XGRAMMAR_PYTHON_DIR}/xgrammar/*.so
+	rm -rf config.cmake
+	echo set\(XGRAMMAR_BUILD_PYTHON_BINDINGS ON\) >>config.cmake
+	echo set\(XGRAMMAR_BUILD_CXX_TESTS OFF\) >>config.cmake
+
+	python3 -m pip install pybind11
+	python3 -m pip install torch --index-url https://download.pytorch.org/whl/cpu
+	rm -rf build
+	mkdir -p build
+	cd build
+	cmake ..
+	make -j4
+	find . -type d -name 'CMakeFiles' -exec rm -rf {} +
+
 	# For compatibility in environments where Conda is not installed,
 	# revert back to previous method of locating cpython_dir.
 	if ! [ -d "${cpython_dir}" ]; then