From d93b8030e708e20af28c7c5af582971ab26c104e Mon Sep 17 00:00:00 2001 From: chuanqiw Date: Wed, 11 Sep 2024 21:26:10 -0700 Subject: [PATCH 1/2] Enable xpu windows in binary build matrix generation --- tools/scripts/generate_binary_build_matrix.py | 5 +++-- .../tests/assets/build_matrix_windows_wheel_xpu.json | 1 + tools/tests/test_generate_binary_build_matrix.py | 11 +++++++++++ 3 files changed, 15 insertions(+), 2 deletions(-) create mode 100644 tools/tests/assets/build_matrix_windows_wheel_xpu.json diff --git a/tools/scripts/generate_binary_build_matrix.py b/tools/scripts/generate_binary_build_matrix.py index be1d1cbffb..ddb99a6053 100644 --- a/tools/scripts/generate_binary_build_matrix.py +++ b/tools/scripts/generate_binary_build_matrix.py @@ -3,11 +3,12 @@ """Generates a matrix to be utilized through github actions Will output a condensed version of the matrix if on a pull request that only -includes the latest version of python we support built on three different +includes the latest version of python we support built on four different architectures: * CPU * Latest CUDA * Latest ROCM + * Latest XPU """ @@ -519,7 +520,7 @@ def generate_wheels_matrix( arches += ROCM_ARCHES if with_xpu == ENABLE: - if os == LINUX: + if os == LINUX or os == WINDOWS: arches += [XPU] if limit_pr_builds: diff --git a/tools/tests/assets/build_matrix_windows_wheel_xpu.json b/tools/tests/assets/build_matrix_windows_wheel_xpu.json new file mode 100644 index 0000000000..c16dff01d5 --- /dev/null +++ b/tools/tests/assets/build_matrix_windows_wheel_xpu.json @@ -0,0 +1 @@ +{"include": [{"python_version": "3.9", "gpu_arch_type": "cpu", "gpu_arch_version": "", "desired_cuda": "cpu", "container_image": "pytorch/manylinux-builder:cpu", "package_type": "wheel", "build_name": "wheel-py3_9-cpu", "validation_runner": "windows.4xlarge", "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cpu", "channel": "nightly", "upload_to_base_bucket": "yes", "stable_version": "2.4.1", "use_split_build": false}, {"python_version": "3.9", "gpu_arch_type": "xpu", "gpu_arch_version": "", "desired_cuda": "xpu", "container_image": "pytorch/manylinux2_28-builder:xpu", "package_type": "wheel", "build_name": "wheel-py3_9-xpu", "validation_runner": "windows.4xlarge", "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/xpu", "channel": "nightly", "upload_to_base_bucket": "yes", "stable_version": "2.4.1", "use_split_build": false}, {"python_version": "3.10", "gpu_arch_type": "cpu", "gpu_arch_version": "", "desired_cuda": "cpu", "container_image": "pytorch/manylinux-builder:cpu", "package_type": "wheel", "build_name": "wheel-py3_10-cpu", "validation_runner": "windows.4xlarge", "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cpu", "channel": "nightly", "upload_to_base_bucket": "yes", "stable_version": "2.4.1", "use_split_build": false}, {"python_version": "3.10", "gpu_arch_type": "xpu", "gpu_arch_version": "", "desired_cuda": "xpu", "container_image": "pytorch/manylinux2_28-builder:xpu", "package_type": "wheel", "build_name": "wheel-py3_10-xpu", "validation_runner": "windows.4xlarge", "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/xpu", "channel": "nightly", "upload_to_base_bucket": "yes", "stable_version": "2.4.1", "use_split_build": false}, {"python_version": "3.11", "gpu_arch_type": "cpu", "gpu_arch_version": "", "desired_cuda": "cpu", "container_image": "pytorch/manylinux-builder:cpu", "package_type": "wheel", "build_name": "wheel-py3_11-cpu", "validation_runner": "windows.4xlarge", "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cpu", "channel": "nightly", "upload_to_base_bucket": "yes", "stable_version": "2.4.1", "use_split_build": false}, {"python_version": "3.11", "gpu_arch_type": "xpu", "gpu_arch_version": "", "desired_cuda": "xpu", "container_image": "pytorch/manylinux2_28-builder:xpu", "package_type": "wheel", "build_name": "wheel-py3_11-xpu", "validation_runner": "windows.4xlarge", "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/xpu", "channel": "nightly", "upload_to_base_bucket": "yes", "stable_version": "2.4.1", "use_split_build": false}, {"python_version": "3.12", "gpu_arch_type": "cpu", "gpu_arch_version": "", "desired_cuda": "cpu", "container_image": "pytorch/manylinux-builder:cpu", "package_type": "wheel", "build_name": "wheel-py3_12-cpu", "validation_runner": "windows.4xlarge", "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cpu", "channel": "nightly", "upload_to_base_bucket": "yes", "stable_version": "2.4.1", "use_split_build": false}, {"python_version": "3.12", "gpu_arch_type": "xpu", "gpu_arch_version": "", "desired_cuda": "xpu", "container_image": "pytorch/manylinux2_28-builder:xpu", "package_type": "wheel", "build_name": "wheel-py3_12-xpu", "validation_runner": "windows.4xlarge", "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/xpu", "channel": "nightly", "upload_to_base_bucket": "yes", "stable_version": "2.4.1", "use_split_build": false}]} diff --git a/tools/tests/test_generate_binary_build_matrix.py b/tools/tests/test_generate_binary_build_matrix.py index 7b09cd2a0a..8c7973ebb3 100644 --- a/tools/tests/test_generate_binary_build_matrix.py +++ b/tools/tests/test_generate_binary_build_matrix.py @@ -108,6 +108,17 @@ def test_windows_conda_cuda(self): reference_output_file="build_matrix_windows_conda_cuda.json", ) + def test_windows_wheel_xpu(self): + self.matrix_compare_helper( + package_type="wheel", + operating_system="windows", + cuda=False, + rocm=False, + cpu=True, + xpu=True, + reference_output_file="build_matrix_windows_wheel_xpu.json", + ) + def test_linux_wheel_cuda_norocm(self): self.matrix_compare_helper( package_type="wheel", From 856c44d6520cf7319846d77d55415c4c4bc6edb9 Mon Sep 17 00:00:00 2001 From: chuanqiw Date: Fri, 20 Sep 2024 11:04:58 -0700 Subject: [PATCH 2/2] enable xpu env for vision/audio CD --- .github/scripts/install_xpu.bat | 91 ++++++++++++++++++++++ .github/workflows/build_wheels_windows.yml | 6 +- 2 files changed, 96 insertions(+), 1 deletion(-) create mode 100644 .github/scripts/install_xpu.bat diff --git a/.github/scripts/install_xpu.bat b/.github/scripts/install_xpu.bat new file mode 100644 index 0000000000..11208580d5 --- /dev/null +++ b/.github/scripts/install_xpu.bat @@ -0,0 +1,91 @@ +@echo on +REM Description: Install Intel Support Packages on Windows +REM BKM reference: https://www.intel.com/content/www/us/en/developer/articles/tool/pytorch-prerequisites-for-intel-gpu/2-5.html + +set XPU_INSTALL_MODE=%~1 +if "%XPU_INSTALL_MODE%"=="" goto xpu_bundle_install_start +if "%XPU_INSTALL_MODE%"=="bundle" goto xpu_bundle_install_start +if "%XPU_INSTALL_MODE%"=="driver" goto xpu_driver_install_start +if "%XPU_INSTALL_MODE%"=="all" goto xpu_driver_install_start + +:arg_error + +echo Illegal XPU installation mode. The value can be "bundle"/"driver"/"all" +echo If keep the value as space, will use default "bundle" mode +exit /b 1 + +:xpu_driver_install_start +:: TODO Need more testing for driver installation +set XPU_DRIVER_LINK=https://downloadmirror.intel.com/830975/gfx_win_101.5972.exe +curl -o xpu_driver.exe --retry 3 --retry-all-errors -k %XPU_DRIVER_LINK% +echo "XPU Driver installing..." +start /wait "Intel XPU Driver Installer" "xpu_driver.exe" +if errorlevel 1 exit /b 1 +del xpu_driver.exe +if "%XPU_INSTALL_MODE%"=="driver" goto xpu_install_end + +:xpu_bundle_install_start + +set XPU_BUNDLE_PARENT_DIR=C:\Program Files (x86)\Intel\oneAPI +set XPU_BUNDLE_URL=https://registrationcenter-download.intel.com/akdlm/IRC_NAS/9d1a91e2-e8b8-40a5-8c7f-5db768a6a60c/w_intel-for-pytorch-gpu-dev_p_0.5.3.37_offline.exe +set XPU_PTI_URL=https://registrationcenter-download.intel.com/akdlm/IRC_NAS/9d1a91e2-e8b8-40a5-8c7f-5db768a6a60c/w_intel-pti-dev_p_0.9.0.37_offline.exe +set XPU_BUNDLE_VERSION=0.5.3+31 +set XPU_PTI_VERSION=0.9.0+36 +set XPU_BUNDLE_PRODUCT_NAME=intel.oneapi.win.intel-for-pytorch-gpu-dev.product +set XPU_PTI_PRODUCT_NAME=intel.oneapi.win.intel-pti-dev.product +set XPU_BUNDLE_INSTALLED=0 +set XPU_PTI_INSTALLED=0 +set XPU_BUNDLE_UNINSTALL=0 +set XPU_PTI_UNINSTALL=0 + +:: Check if XPU bundle is target version or already installed +if exist "%XPU_BUNDLE_PARENT_DIR%\Installer\installer.exe" goto xpu_bundle_ver_check +goto xpu_bundle_install + +:xpu_bundle_ver_check + +"%XPU_BUNDLE_PARENT_DIR%\Installer\installer.exe" --list-products > xpu_bundle_installed_ver.log + +for /f "tokens=1,2" %%a in (xpu_bundle_installed_ver.log) do ( + if "%%a"=="%XPU_BUNDLE_PRODUCT_NAME%" ( + echo %%a Installed Version: %%b + set XPU_BUNDLE_INSTALLED=1 + if not "%XPU_BUNDLE_VERSION%"=="%%b" ( + start /wait "Installer Title" "%XPU_BUNDLE_PARENT_DIR%\Installer\installer.exe" --action=remove --eula=accept --silent --product-id %XPU_BUNDLE_PRODUCT_NAME% --product-ver %%b --log-dir uninstall_bundle + set XPU_BUNDLE_UNINSTALL=1 + ) + ) + if "%%a"=="%XPU_PTI_PRODUCT_NAME%" ( + echo %%a Installed Version: %%b + set XPU_PTI_INSTALLED=1 + if not "%XPU_PTI_VERSION%"=="%%b" ( + start /wait "Installer Title" "%XPU_BUNDLE_PARENT_DIR%\Installer\installer.exe" --action=remove --eula=accept --silent --product-id %XPU_PTI_PRODUCT_NAME% --product-ver %%b --log-dir uninstall_bundle + set XPU_PTI_UNINSTALL=1 + ) + ) +) +if errorlevel 1 exit /b 1 +if exist xpu_bundle_installed_ver.log del xpu_bundle_installed_ver.log +if "%XPU_BUNDLE_INSTALLED%"=="0" goto xpu_bundle_install +if "%XPU_BUNDLE_UNINSTALL%"=="1" goto xpu_bundle_install +if "%XPU_PTI_INSTALLED%"=="0" goto xpu_pti_install +if "%XPU_PTI_UNINSTALL%"=="1" goto xpu_pti_install +goto xpu_install_end + +:xpu_bundle_install + +curl -o xpu_bundle.exe --retry 3 --retry-all-errors -k %XPU_BUNDLE_URL% +echo "XPU Bundle installing..." +start /wait "Intel Pytorch Bundle Installer" "xpu_bundle.exe" --action=install --eula=accept --silent --log-dir install_bundle +if errorlevel 1 exit /b 1 +del xpu_bundle.exe + +:xpu_pti_install + +curl -o xpu_pti.exe --retry 3 --retry-all-errors -k %XPU_PTI_URL% +echo "XPU PTI installing..." +start /wait "Intel PTI Installer" "xpu_pti.exe" --action=install --eula=accept --silent --log-dir install_bundle +if errorlevel 1 exit /b 1 +del xpu_pti.exe + +:xpu_install_end diff --git a/.github/workflows/build_wheels_windows.yml b/.github/workflows/build_wheels_windows.yml index 2c062a6e70..635aa0c96a 100644 --- a/.github/workflows/build_wheels_windows.yml +++ b/.github/workflows/build_wheels_windows.yml @@ -121,6 +121,10 @@ jobs: python-version: ${{ env.PYTHON_VERSION }} cuda-version: ${{ env.CU_VERSION }} arch: ${{ env.ARCH }} + - name: Install XPU support package + if: ${{ matrix.gpu_arch_type == 'xpu' }} + run: | + cmd "/c .\\test-infra\\.github\\scripts\\install_xpu.bat" - name: Install torch dependency run: | source "${BUILD_ENV_FILE}" @@ -190,7 +194,7 @@ jobs: ${CONDA_RUN} python -c "import ${PACKAGE_NAME}; print('package version is ', ${PACKAGE_NAME}.__version__)" else echo "${{ inputs.repository }}/${SMOKE_TEST_SCRIPT} found" - ${CONDA_RUN} python "${{ inputs.repository }}/${SMOKE_TEST_SCRIPT}" + ${CONDA_RUN} ${ENV_SCRIPT} python "${{ inputs.repository }}/${SMOKE_TEST_SCRIPT}" fi # NB: Only upload to GitHub after passing smoke tests - name: Upload wheel to GitHub