diff --git a/.github/workflows/build_wheels_linux.yml b/.github/workflows/build_wheels_linux.yml index b789b169e6..e3238a98ed 100644 --- a/.github/workflows/build_wheels_linux.yml +++ b/.github/workflows/build_wheels_linux.yml @@ -171,6 +171,11 @@ jobs: run: | set -euxo pipefail cat "${{ inputs.env-var-script }}" >> "${BUILD_ENV_FILE}" + - name: Add XPU Env Vars in Build Env File + if: ${{ matrix.gpu_arch_type == 'xpu' }} + run: | + echo "set +u" >> "${BUILD_ENV_FILE}" + echo "source /opt/intel/oneapi/pytorch-gpu-dev-0.5/oneapi-vars.sh" >> "${BUILD_ENV_FILE}" - name: Install torch dependency run: | set -euxo pipefail diff --git a/.github/workflows/generate_binary_build_matrix.yml b/.github/workflows/generate_binary_build_matrix.yml index c9ba6e9efe..9ec18c960d 100644 --- a/.github/workflows/generate_binary_build_matrix.yml +++ b/.github/workflows/generate_binary_build_matrix.yml @@ -35,6 +35,10 @@ on: description: "Build with CPU?" default: "enable" type: string + with-xpu: + description: "Build with XPU?" + default: "disable" + type: string use-only-dl-pytorch-org: description: "Use only download.pytorch.org when generating wheel install command?" default: "false" @@ -80,6 +84,7 @@ jobs: WITH_CUDA: ${{ inputs.with-cuda }} WITH_ROCM: ${{ inputs.with-rocm }} WITH_CPU: ${{ inputs.with-cpu }} + WITH_XPU: ${{ inputs.with-xpu }} # limit pull request builds to one version of python unless ciflow/binaries/all is applied to the workflow # should not affect builds that are from events that are not the pull_request event LIMIT_PR_BUILDS: ${{ github.event_name == 'pull_request' && !contains( github.event.pull_request.labels.*.name, 'ciflow/binaries/all') }} diff --git a/tools/scripts/generate_binary_build_matrix.py b/tools/scripts/generate_binary_build_matrix.py index c108d6a122..62415811e7 100644 --- a/tools/scripts/generate_binary_build_matrix.py +++ b/tools/scripts/generate_binary_build_matrix.py @@ -61,6 +61,7 @@ CUDA_AARCH64 = "cuda-aarch64" CUDA = "cuda" ROCM = "rocm" +XPU = "xpu" CURRENT_NIGHTLY_VERSION = "2.5.0" @@ -107,6 +108,8 @@ def arch_type(arch_version: str) -> str: return CPU_AARCH64 elif arch_version == CUDA_AARCH64: return CUDA_AARCH64 + elif arch_version == XPU: + return XPU else: # arch_version should always be CPU in this case return CPU @@ -160,6 +163,7 @@ def initialize_globals(channel: str, build_python_only: bool) -> None: for gpu_arch in ROCM_ARCHES }, CPU: "pytorch/manylinux-builder:cpu", + XPU: "pytorch/manylinux2_28-builder:xpu", CPU_AARCH64: "pytorch/manylinuxaarch64-builder:cpu-aarch64", CUDA_AARCH64: "pytorch/manylinuxaarch64-builder:cuda12.4", } @@ -199,6 +203,7 @@ def translate_desired_cuda(gpu_arch_type: str, gpu_arch_version: str) -> str: CUDA_AARCH64: "cu124", CUDA: f"cu{gpu_arch_version.replace('.', '')}", ROCM: f"rocm{gpu_arch_version}", + XPU: "xpu", }.get(gpu_arch_type, gpu_arch_version) @@ -327,6 +332,7 @@ def generate_conda_matrix( with_cuda: str, with_rocm: str, with_cpu: str, + with_xpu: str, limit_pr_builds: bool, use_only_dl_pytorch_org: bool, use_split_build: bool = False, @@ -383,6 +389,7 @@ def generate_libtorch_matrix( with_cuda: str, with_rocm: str, with_cpu: str, + with_xpu: str, limit_pr_builds: bool, use_only_dl_pytorch_org: bool, use_split_build: bool = False, @@ -472,6 +479,7 @@ def generate_wheels_matrix( with_cuda: str, with_rocm: str, with_cpu: str, + with_xpu: str, limit_pr_builds: bool, use_only_dl_pytorch_org: bool, use_split_build: bool = False, @@ -510,6 +518,10 @@ def generate_wheels_matrix( if os == LINUX: arches += ROCM_ARCHES + if with_xpu == ENABLE: + if os == LINUX: + arches += [XPU] + if limit_pr_builds: python_versions = [python_versions[0]] @@ -523,7 +535,7 @@ def generate_wheels_matrix( continue gpu_arch_version = ( "" - if arch_version in [CPU, CPU_AARCH64] + if arch_version in [CPU, CPU_AARCH64, XPU] else arch_version ) @@ -579,6 +591,7 @@ def generate_build_matrix( with_cuda: str, with_rocm: str, with_cpu: str, + with_xpu: str, limit_pr_builds: str, use_only_dl_pytorch_org: str, build_python_only: str, @@ -602,6 +615,7 @@ def generate_build_matrix( with_cuda, with_rocm, with_cpu, + with_xpu, limit_pr_builds == "true", use_only_dl_pytorch_org == "true", use_split_build == "true", @@ -653,6 +667,13 @@ def main(args: List[str]) -> None: choices=[ENABLE, DISABLE], default=os.getenv("WITH_CPU", ENABLE), ) + parser.add_argument( + "--with-xpu", + help="Build with XPU?", + type=str, + choices=[ENABLE, DISABLE], + default=os.getenv("WITH_XPU", ENABLE), + ) # By default this is false for this script but expectation is that the caller # workflow will default this to be true most of the time, where a pull # request is synchronized and does not contain the label "ciflow/binaries/all" @@ -695,8 +716,8 @@ def main(args: List[str]) -> None: options = parser.parse_args(args) assert ( - options.with_cuda or options.with_rocm or options.with_cpu - ), "Must build with either CUDA, ROCM, or CPU support." + options.with_cuda or options.with_rocm or options.with_xpu or options.with_cpu + ), "Must build with either CUDA, ROCM, XPU, or CPU support." build_matrix = generate_build_matrix( options.package_type, @@ -705,6 +726,7 @@ def main(args: List[str]) -> None: options.with_cuda, options.with_rocm, options.with_cpu, + options.with_xpu, options.limit_pr_builds, options.use_only_dl_pytorch_org, options.build_python_only, diff --git a/tools/tests/assets/build_matrix_linux_wheel_xpu.json b/tools/tests/assets/build_matrix_linux_wheel_xpu.json new file mode 100644 index 0000000000..bee24301c3 --- /dev/null +++ b/tools/tests/assets/build_matrix_linux_wheel_xpu.json @@ -0,0 +1 @@ +{"include": [{"python_version": "3.8", "gpu_arch_type": "cuda", "gpu_arch_version": "11.8", "desired_cuda": "cu118", "container_image": "pytorch/manylinux-builder:cuda11.8", "package_type": "manywheel", "build_name": "manywheel-py3_8-cuda11_8", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu118", "channel": "nightly", "upload_to_base_bucket": "no", "stable_version": "2.4.0", "use_split_build": false}, {"python_version": "3.8", "gpu_arch_type": "cuda", "gpu_arch_version": "12.1", "desired_cuda": "cu121", "container_image": "pytorch/manylinux-builder:cuda12.1", "package_type": "manywheel", "build_name": "manywheel-py3_8-cuda12_1", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu121", "channel": "nightly", "upload_to_base_bucket": "no", "stable_version": "2.4.0", "use_split_build": false}, {"python_version": "3.8", "gpu_arch_type": "cuda", "gpu_arch_version": "12.4", "desired_cuda": "cu124", "container_image": "pytorch/manylinux-builder:cuda12.4", "package_type": "manywheel", "build_name": "manywheel-py3_8-cuda12_4", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu124", "channel": "nightly", "upload_to_base_bucket": "no", "stable_version": "2.4.0", "use_split_build": false}, {"python_version": "3.8", "gpu_arch_type": "xpu", "gpu_arch_version": "", "desired_cuda": "xpu", "container_image": "pytorch/manylinux2_28-builder:xpu", "package_type": "manywheel", "build_name": "manywheel-py3_8-xpu", "validation_runner": "linux.2xlarge", "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/xpu", "channel": "nightly", "upload_to_base_bucket": "no", "stable_version": "2.4.0", "use_split_build": false}, {"python_version": "3.9", "gpu_arch_type": "cuda", "gpu_arch_version": "11.8", "desired_cuda": "cu118", "container_image": "pytorch/manylinux-builder:cuda11.8", "package_type": "manywheel", "build_name": "manywheel-py3_9-cuda11_8", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu118", "channel": "nightly", "upload_to_base_bucket": "no", "stable_version": "2.4.0", "use_split_build": false}, {"python_version": "3.9", "gpu_arch_type": "cuda", "gpu_arch_version": "12.1", "desired_cuda": "cu121", "container_image": "pytorch/manylinux-builder:cuda12.1", "package_type": "manywheel", "build_name": "manywheel-py3_9-cuda12_1", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu121", "channel": "nightly", "upload_to_base_bucket": "no", "stable_version": "2.4.0", "use_split_build": false}, {"python_version": "3.9", "gpu_arch_type": "cuda", "gpu_arch_version": "12.4", "desired_cuda": "cu124", "container_image": "pytorch/manylinux-builder:cuda12.4", "package_type": "manywheel", "build_name": "manywheel-py3_9-cuda12_4", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu124", "channel": "nightly", "upload_to_base_bucket": "no", "stable_version": "2.4.0", "use_split_build": false}, {"python_version": "3.9", "gpu_arch_type": "xpu", "gpu_arch_version": "", "desired_cuda": "xpu", "container_image": "pytorch/manylinux2_28-builder:xpu", "package_type": "manywheel", "build_name": "manywheel-py3_9-xpu", "validation_runner": "linux.2xlarge", "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/xpu", "channel": "nightly", "upload_to_base_bucket": "no", "stable_version": "2.4.0", "use_split_build": false}, {"python_version": "3.10", "gpu_arch_type": "cuda", "gpu_arch_version": "11.8", "desired_cuda": "cu118", "container_image": "pytorch/manylinux-builder:cuda11.8", "package_type": "manywheel", "build_name": "manywheel-py3_10-cuda11_8", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu118", "channel": "nightly", "upload_to_base_bucket": "no", "stable_version": "2.4.0", "use_split_build": false}, {"python_version": "3.10", "gpu_arch_type": "cuda", "gpu_arch_version": "12.1", "desired_cuda": "cu121", "container_image": "pytorch/manylinux-builder:cuda12.1", "package_type": "manywheel", "build_name": "manywheel-py3_10-cuda12_1", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu121", "channel": "nightly", "upload_to_base_bucket": "no", "stable_version": "2.4.0", "use_split_build": false}, {"python_version": "3.10", "gpu_arch_type": "cuda", "gpu_arch_version": "12.4", "desired_cuda": "cu124", "container_image": "pytorch/manylinux-builder:cuda12.4", "package_type": "manywheel", "build_name": "manywheel-py3_10-cuda12_4", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu124", "channel": "nightly", "upload_to_base_bucket": "no", "stable_version": "2.4.0", "use_split_build": false}, {"python_version": "3.10", "gpu_arch_type": "xpu", "gpu_arch_version": "", "desired_cuda": "xpu", "container_image": "pytorch/manylinux2_28-builder:xpu", "package_type": "manywheel", "build_name": "manywheel-py3_10-xpu", "validation_runner": "linux.2xlarge", "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/xpu", "channel": "nightly", "upload_to_base_bucket": "no", "stable_version": "2.4.0", "use_split_build": false}, {"python_version": "3.11", "gpu_arch_type": "cuda", "gpu_arch_version": "11.8", "desired_cuda": "cu118", "container_image": "pytorch/manylinux-builder:cuda11.8", "package_type": "manywheel", "build_name": "manywheel-py3_11-cuda11_8", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu118", "channel": "nightly", "upload_to_base_bucket": "no", "stable_version": "2.4.0", "use_split_build": false}, {"python_version": "3.11", "gpu_arch_type": "cuda", "gpu_arch_version": "12.1", "desired_cuda": "cu121", "container_image": "pytorch/manylinux-builder:cuda12.1", "package_type": "manywheel", "build_name": "manywheel-py3_11-cuda12_1", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu121", "channel": "nightly", "upload_to_base_bucket": "no", "stable_version": "2.4.0", "use_split_build": false}, {"python_version": "3.11", "gpu_arch_type": "cuda", "gpu_arch_version": "12.4", "desired_cuda": "cu124", "container_image": "pytorch/manylinux-builder:cuda12.4", "package_type": "manywheel", "build_name": "manywheel-py3_11-cuda12_4", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu124", "channel": "nightly", "upload_to_base_bucket": "no", "stable_version": "2.4.0", "use_split_build": false}, {"python_version": "3.11", "gpu_arch_type": "xpu", "gpu_arch_version": "", "desired_cuda": "xpu", "container_image": "pytorch/manylinux2_28-builder:xpu", "package_type": "manywheel", "build_name": "manywheel-py3_11-xpu", "validation_runner": "linux.2xlarge", "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/xpu", "channel": "nightly", "upload_to_base_bucket": "no", "stable_version": "2.4.0", "use_split_build": false}, {"python_version": "3.12", "gpu_arch_type": "cuda", "gpu_arch_version": "11.8", "desired_cuda": "cu118", "container_image": "pytorch/manylinux-builder:cuda11.8", "package_type": "manywheel", "build_name": "manywheel-py3_12-cuda11_8", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu118", "channel": "nightly", "upload_to_base_bucket": "no", "stable_version": "2.4.0", "use_split_build": false}, {"python_version": "3.12", "gpu_arch_type": "cuda", "gpu_arch_version": "12.1", "desired_cuda": "cu121", "container_image": "pytorch/manylinux-builder:cuda12.1", "package_type": "manywheel", "build_name": "manywheel-py3_12-cuda12_1", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu121", "channel": "nightly", "upload_to_base_bucket": "no", "stable_version": "2.4.0", "use_split_build": false}, {"python_version": "3.12", "gpu_arch_type": "cuda", "gpu_arch_version": "12.4", "desired_cuda": "cu124", "container_image": "pytorch/manylinux-builder:cuda12.4", "package_type": "manywheel", "build_name": "manywheel-py3_12-cuda12_4", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu124", "channel": "nightly", "upload_to_base_bucket": "no", "stable_version": "2.4.0", "use_split_build": false}, {"python_version": "3.12", "gpu_arch_type": "xpu", "gpu_arch_version": "", "desired_cuda": "xpu", "container_image": "pytorch/manylinux2_28-builder:xpu", "package_type": "manywheel", "build_name": "manywheel-py3_12-xpu", "validation_runner": "linux.2xlarge", "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/xpu", "channel": "nightly", "upload_to_base_bucket": "no", "stable_version": "2.4.0", "use_split_build": false}]} diff --git a/tools/tests/test_generate_binary_build_matrix.py b/tools/tests/test_generate_binary_build_matrix.py index 3753f4da27..7b09cd2a0a 100644 --- a/tools/tests/test_generate_binary_build_matrix.py +++ b/tools/tests/test_generate_binary_build_matrix.py @@ -16,6 +16,7 @@ def matrix_compare_helper( cuda: bool, rocm: bool, cpu: bool, + xpu: bool, reference_output_file: str, build_python_only: bool = False, ) -> None: @@ -26,6 +27,7 @@ def matrix_compare_helper( "enable" if cuda else "disable", "enable" if rocm else "disable", "enable" if cpu else "disable", + "enable" if xpu else "disable", "false", "false", "enable" if build_python_only else "disable", @@ -47,6 +49,7 @@ def test_linux_wheel_cuda(self): cuda=True, rocm=True, cpu=True, + xpu=False, reference_output_file="build_matrix_linux_wheel_cuda.json", ) @@ -57,6 +60,7 @@ def test_linux_conda_cuda(self): cuda=True, rocm=True, cpu=True, + xpu=False, reference_output_file="build_matrix_linux_conda_cuda.json", ) @@ -67,6 +71,7 @@ def test_macos_wheel(self): cuda=False, rocm=False, cpu=True, + xpu=False, reference_output_file="build_matrix_macos_wheel.json", ) @@ -77,6 +82,7 @@ def test_macos_conda(self): cuda=False, rocm=False, cpu=True, + xpu=False, reference_output_file="build_matrix_macos_conda.json", ) @@ -87,6 +93,7 @@ def test_windows_wheel_cuda(self): cuda=True, rocm=True, cpu=True, + xpu=False, reference_output_file="build_matrix_windows_wheel_cuda.json", ) @@ -97,6 +104,7 @@ def test_windows_conda_cuda(self): cuda=True, rocm=True, cpu=True, + xpu=True, reference_output_file="build_matrix_windows_conda_cuda.json", ) @@ -107,6 +115,7 @@ def test_linux_wheel_cuda_norocm(self): cuda=True, rocm=False, cpu=True, + xpu=False, reference_output_file="build_matrix_linux_wheel_cuda_norocm.json", ) @@ -117,8 +126,20 @@ def test_linux_wheel_cuda_rocm_nocpu(self): cuda=True, rocm=True, cpu=False, + xpu=False, reference_output_file="build_matrix_linux_wheel_nocpu.json", ) + + def test_linux_wheel_cuda_xpu_nocpu(self): + self.matrix_compare_helper( + package_type="wheel", + operating_system="linux", + cuda=True, + rocm=False, + cpu=False, + xpu=True, + reference_output_file="build_matrix_linux_wheel_xpu.json", + ) if __name__ == "__main__":