Add XPU to binary build generation

.github/workflows/build_wheels_linux.yml

@@ -171,6 +171,11 @@ jobs:
         run: |
           set -euxo pipefail
           cat "${{ inputs.env-var-script }}" >> "${BUILD_ENV_FILE}"
+      - name: Add XPU Env Vars in Build Env File
+        if: ${{ matrix.gpu_arch_type == 'xpu' }}
+        run: |
+          echo "set +u" >> "${BUILD_ENV_FILE}"
+          echo "source /opt/intel/oneapi/pytorch-gpu-dev-0.5/oneapi-vars.sh" >> "${BUILD_ENV_FILE}"
       - name: Install torch dependency
         run: |
           set -euxo pipefail

.github/workflows/generate_binary_build_matrix.yml

@@ -35,6 +35,10 @@ on:
         description: "Build with CPU?"
         default: "enable"
         type: string
+      with-xpu:
+        description: "Build with XPU?"
+        default: "disable"
+        type: string
       use-only-dl-pytorch-org:
         description: "Use only download.pytorch.org when generating wheel install command?"
         default: "false"
@@ -80,6 +84,7 @@ jobs:
           WITH_CUDA: ${{ inputs.with-cuda }}
           WITH_ROCM: ${{ inputs.with-rocm }}
           WITH_CPU: ${{ inputs.with-cpu }}
+          WITH_XPU: ${{ inputs.with-xpu }}
           # limit pull request builds to one version of python unless ciflow/binaries/all is applied to the workflow
           # should not affect builds that are from events that are not the pull_request event
           LIMIT_PR_BUILDS: ${{ github.event_name == 'pull_request' && !contains( github.event.pull_request.labels.*.name, 'ciflow/binaries/all') }}

tools/scripts/generate_binary_build_matrix.py

@@ -61,6 +61,7 @@
 CUDA_AARCH64 = "cuda-aarch64"
 CUDA = "cuda"
 ROCM = "rocm"
+XPU = "xpu"
 
 
 CURRENT_NIGHTLY_VERSION = "2.5.0"
@@ -107,6 +108,8 @@ def arch_type(arch_version: str) -> str:
         return CPU_AARCH64
     elif arch_version == CUDA_AARCH64:
         return CUDA_AARCH64
+    elif arch_version == XPU:
+        return XPU
     else:  # arch_version should always be CPU in this case
         return CPU
 
@@ -160,6 +163,7 @@ def initialize_globals(channel: str, build_python_only: bool) -> None:
             for gpu_arch in ROCM_ARCHES
         },
         CPU: "pytorch/manylinux-builder:cpu",
+        XPU: "pytorch/manylinux2_28-builder:xpu",
         CPU_AARCH64: "pytorch/manylinuxaarch64-builder:cpu-aarch64",
         CUDA_AARCH64: "pytorch/manylinuxaarch64-builder:cuda12.4",
     }
@@ -199,6 +203,7 @@ def translate_desired_cuda(gpu_arch_type: str, gpu_arch_version: str) -> str:
         CUDA_AARCH64: "cu124",
         CUDA: f"cu{gpu_arch_version.replace('.', '')}",
         ROCM: f"rocm{gpu_arch_version}",
+        XPU: "xpu",
     }.get(gpu_arch_type, gpu_arch_version)
 
 
@@ -327,6 +332,7 @@ def generate_conda_matrix(
     with_cuda: str,
     with_rocm: str,
     with_cpu: str,
+    with_xpu: str,
     limit_pr_builds: bool,
     use_only_dl_pytorch_org: bool,
     use_split_build: bool = False,
@@ -383,6 +389,7 @@ def generate_libtorch_matrix(
     with_cuda: str,
     with_rocm: str,
     with_cpu: str,
+    with_xpu: str,
     limit_pr_builds: bool,
     use_only_dl_pytorch_org: bool,
     use_split_build: bool = False,
@@ -472,6 +479,7 @@ def generate_wheels_matrix(
     with_cuda: str,
     with_rocm: str,
     with_cpu: str,
+    with_xpu: str,
     limit_pr_builds: bool,
     use_only_dl_pytorch_org: bool,
     use_split_build: bool = False,
@@ -510,6 +518,10 @@ def generate_wheels_matrix(
             if os == LINUX:
                 arches += ROCM_ARCHES
 
+        if with_xpu == ENABLE:
+            if os == LINUX:
+                arches += [XPU]
+
     if limit_pr_builds:
         python_versions = [python_versions[0]]
 
@@ -523,7 +535,7 @@ def generate_wheels_matrix(
                 continue
             gpu_arch_version = (
                 ""
-                if arch_version in [CPU, CPU_AARCH64]
+                if arch_version in [CPU, CPU_AARCH64, XPU]
                 else arch_version
             )
 
@@ -579,6 +591,7 @@ def generate_build_matrix(
     with_cuda: str,
     with_rocm: str,
     with_cpu: str,
+    with_xpu: str,
     limit_pr_builds: str,
     use_only_dl_pytorch_org: str,
     build_python_only: str,
@@ -602,6 +615,7 @@ def generate_build_matrix(
                     with_cuda,
                     with_rocm,
                     with_cpu,
+                    with_xpu,
                     limit_pr_builds == "true",
                     use_only_dl_pytorch_org == "true",
                     use_split_build == "true",
@@ -653,6 +667,13 @@ def main(args: List[str]) -> None:
         choices=[ENABLE, DISABLE],
         default=os.getenv("WITH_CPU", ENABLE),
     )
+    parser.add_argument(
+        "--with-xpu",
+        help="Build with XPU?",
+        type=str,
+        choices=[ENABLE, DISABLE],
+        default=os.getenv("WITH_XPU", ENABLE),
+    )
     # By default this is false for this script but expectation is that the caller
     # workflow will default this to be true most of the time, where a pull
     # request is synchronized and does not contain the label "ciflow/binaries/all"
@@ -695,8 +716,8 @@ def main(args: List[str]) -> None:
     options = parser.parse_args(args)
 
     assert (
-        options.with_cuda or options.with_rocm or options.with_cpu
-    ), "Must build with either CUDA, ROCM, or CPU support."
+        options.with_cuda or options.with_rocm or options.with_xpu or options.with_cpu
+    ), "Must build with either CUDA, ROCM, XPU, or CPU support."
 
     build_matrix = generate_build_matrix(
         options.package_type,
@@ -705,6 +726,7 @@ def main(args: List[str]) -> None:
         options.with_cuda,
         options.with_rocm,
         options.with_cpu,
+        options.with_xpu,
         options.limit_pr_builds,
         options.use_only_dl_pytorch_org,
         options.build_python_only,

tools/tests/assets/build_matrix_linux_wheel_xpu.json

@@ -0,0 +1 @@
+{"include": [{"python_version": "3.8", "gpu_arch_type": "cuda", "gpu_arch_version": "11.8", "desired_cuda": "cu118", "container_image": "pytorch/manylinux-builder:cuda11.8", "package_type": "manywheel", "build_name": "manywheel-py3_8-cuda11_8", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu118", "channel": "nightly", "upload_to_base_bucket": "no", "stable_version": "2.4.0", "use_split_build": false}, {"python_version": "3.8", "gpu_arch_type": "cuda", "gpu_arch_version": "12.1", "desired_cuda": "cu121", "container_image": "pytorch/manylinux-builder:cuda12.1", "package_type": "manywheel", "build_name": "manywheel-py3_8-cuda12_1", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu121", "channel": "nightly", "upload_to_base_bucket": "no", "stable_version": "2.4.0", "use_split_build": false}, {"python_version": "3.8", "gpu_arch_type": "cuda", "gpu_arch_version": "12.4", "desired_cuda": "cu124", "container_image": "pytorch/manylinux-builder:cuda12.4", "package_type": "manywheel", "build_name": "manywheel-py3_8-cuda12_4", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu124", "channel": "nightly", "upload_to_base_bucket": "no", "stable_version": "2.4.0", "use_split_build": false}, {"python_version": "3.8", "gpu_arch_type": "xpu", "gpu_arch_version": "", "desired_cuda": "xpu", "container_image": "pytorch/manylinux2_28-builder:xpu", "package_type": "manywheel", "build_name": "manywheel-py3_8-xpu", "validation_runner": "linux.2xlarge", "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/xpu", "channel": "nightly", "upload_to_base_bucket": "no", "stable_version": "2.4.0", "use_split_build": false}, {"python_version": "3.9", "gpu_arch_type": "cuda", "gpu_arch_version": "11.8", "desired_cuda": "cu118", "container_image": "pytorch/manylinux-builder:cuda11.8", "package_type": "manywheel", "build_name": "manywheel-py3_9-cuda11_8", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu118", "channel": "nightly", "upload_to_base_bucket": "no", "stable_version": "2.4.0", "use_split_build": false}, {"python_version": "3.9", "gpu_arch_type": "cuda", "gpu_arch_version": "12.1", "desired_cuda": "cu121", "container_image": "pytorch/manylinux-builder:cuda12.1", "package_type": "manywheel", "build_name": "manywheel-py3_9-cuda12_1", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu121", "channel": "nightly", "upload_to_base_bucket": "no", "stable_version": "2.4.0", "use_split_build": false}, {"python_version": "3.9", "gpu_arch_type": "cuda", "gpu_arch_version": "12.4", "desired_cuda": "cu124", "container_image": "pytorch/manylinux-builder:cuda12.4", "package_type": "manywheel", "build_name": "manywheel-py3_9-cuda12_4", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu124", "channel": "nightly", "upload_to_base_bucket": "no", "stable_version": "2.4.0", "use_split_build": false}, {"python_version": "3.9", "gpu_arch_type": "xpu", "gpu_arch_version": "", "desired_cuda": "xpu", "container_image": "pytorch/manylinux2_28-builder:xpu", "package_type": "manywheel", "build_name": "manywheel-py3_9-xpu", "validation_runner": "linux.2xlarge", "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/xpu", "channel": "nightly", "upload_to_base_bucket": "no", "stable_version": "2.4.0", "use_split_build": false}, {"python_version": "3.10", "gpu_arch_type": "cuda", "gpu_arch_version": "11.8", "desired_cuda": "cu118", "container_image": "pytorch/manylinux-builder:cuda11.8", "package_type": "manywheel", "build_name": "manywheel-py3_10-cuda11_8", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu118", "channel": "nightly", "upload_to_base_bucket": "no", "stable_version": "2.4.0", "use_split_build": false}, {"python_version": "3.10", "gpu_arch_type": "cuda", "gpu_arch_version": "12.1", "desired_cuda": "cu121", "container_image": "pytorch/manylinux-builder:cuda12.1", "package_type": "manywheel", "build_name": "manywheel-py3_10-cuda12_1", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu121", "channel": "nightly", "upload_to_base_bucket": "no", "stable_version": "2.4.0", "use_split_build": false}, {"python_version": "3.10", "gpu_arch_type": "cuda", "gpu_arch_version": "12.4", "desired_cuda": "cu124", "container_image": "pytorch/manylinux-builder:cuda12.4", "package_type": "manywheel", "build_name": "manywheel-py3_10-cuda12_4", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu124", "channel": "nightly", "upload_to_base_bucket": "no", "stable_version": "2.4.0", "use_split_build": false}, {"python_version": "3.10", "gpu_arch_type": "xpu", "gpu_arch_version": "", "desired_cuda": "xpu", "container_image": "pytorch/manylinux2_28-builder:xpu", "package_type": "manywheel", "build_name": "manywheel-py3_10-xpu", "validation_runner": "linux.2xlarge", "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/xpu", "channel": "nightly", "upload_to_base_bucket": "no", "stable_version": "2.4.0", "use_split_build": false}, {"python_version": "3.11", "gpu_arch_type": "cuda", "gpu_arch_version": "11.8", "desired_cuda": "cu118", "container_image": "pytorch/manylinux-builder:cuda11.8", "package_type": "manywheel", "build_name": "manywheel-py3_11-cuda11_8", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu118", "channel": "nightly", "upload_to_base_bucket": "no", "stable_version": "2.4.0", "use_split_build": false}, {"python_version": "3.11", "gpu_arch_type": "cuda", "gpu_arch_version": "12.1", "desired_cuda": "cu121", "container_image": "pytorch/manylinux-builder:cuda12.1", "package_type": "manywheel", "build_name": "manywheel-py3_11-cuda12_1", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu121", "channel": "nightly", "upload_to_base_bucket": "no", "stable_version": "2.4.0", "use_split_build": false}, {"python_version": "3.11", "gpu_arch_type": "cuda", "gpu_arch_version": "12.4", "desired_cuda": "cu124", "container_image": "pytorch/manylinux-builder:cuda12.4", "package_type": "manywheel", "build_name": "manywheel-py3_11-cuda12_4", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu124", "channel": "nightly", "upload_to_base_bucket": "no", "stable_version": "2.4.0", "use_split_build": false}, {"python_version": "3.11", "gpu_arch_type": "xpu", "gpu_arch_version": "", "desired_cuda": "xpu", "container_image": "pytorch/manylinux2_28-builder:xpu", "package_type": "manywheel", "build_name": "manywheel-py3_11-xpu", "validation_runner": "linux.2xlarge", "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/xpu", "channel": "nightly", "upload_to_base_bucket": "no", "stable_version": "2.4.0", "use_split_build": false}, {"python_version": "3.12", "gpu_arch_type": "cuda", "gpu_arch_version": "11.8", "desired_cuda": "cu118", "container_image": "pytorch/manylinux-builder:cuda11.8", "package_type": "manywheel", "build_name": "manywheel-py3_12-cuda11_8", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu118", "channel": "nightly", "upload_to_base_bucket": "no", "stable_version": "2.4.0", "use_split_build": false}, {"python_version": "3.12", "gpu_arch_type": "cuda", "gpu_arch_version": "12.1", "desired_cuda": "cu121", "container_image": "pytorch/manylinux-builder:cuda12.1", "package_type": "manywheel", "build_name": "manywheel-py3_12-cuda12_1", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu121", "channel": "nightly", "upload_to_base_bucket": "no", "stable_version": "2.4.0", "use_split_build": false}, {"python_version": "3.12", "gpu_arch_type": "cuda", "gpu_arch_version": "12.4", "desired_cuda": "cu124", "container_image": "pytorch/manylinux-builder:cuda12.4", "package_type": "manywheel", "build_name": "manywheel-py3_12-cuda12_4", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu124", "channel": "nightly", "upload_to_base_bucket": "no", "stable_version": "2.4.0", "use_split_build": false}, {"python_version": "3.12", "gpu_arch_type": "xpu", "gpu_arch_version": "", "desired_cuda": "xpu", "container_image": "pytorch/manylinux2_28-builder:xpu", "package_type": "manywheel", "build_name": "manywheel-py3_12-xpu", "validation_runner": "linux.2xlarge", "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/xpu", "channel": "nightly", "upload_to_base_bucket": "no", "stable_version": "2.4.0", "use_split_build": false}]}

tools/tests/test_generate_binary_build_matrix.py

@@ -16,6 +16,7 @@ def matrix_compare_helper(
         cuda: bool,
         rocm: bool,
         cpu: bool,
+        xpu: bool,
         reference_output_file: str,
         build_python_only: bool = False,
     ) -> None:
@@ -26,6 +27,7 @@ def matrix_compare_helper(
             "enable" if cuda else "disable",
             "enable" if rocm else "disable",
             "enable" if cpu else "disable",
+            "enable" if xpu else "disable",
             "false",
             "false",
             "enable" if build_python_only else "disable",
@@ -47,6 +49,7 @@ def test_linux_wheel_cuda(self):
             cuda=True,
             rocm=True,
             cpu=True,
+            xpu=False,
             reference_output_file="build_matrix_linux_wheel_cuda.json",
         )
 
@@ -57,6 +60,7 @@ def test_linux_conda_cuda(self):
             cuda=True,
             rocm=True,
             cpu=True,
+            xpu=False,
             reference_output_file="build_matrix_linux_conda_cuda.json",
         )
 
@@ -67,6 +71,7 @@ def test_macos_wheel(self):
             cuda=False,
             rocm=False,
             cpu=True,
+            xpu=False,
             reference_output_file="build_matrix_macos_wheel.json",
         )
 
@@ -77,6 +82,7 @@ def test_macos_conda(self):
             cuda=False,
             rocm=False,
             cpu=True,
+            xpu=False,
             reference_output_file="build_matrix_macos_conda.json",
         )
 
@@ -87,6 +93,7 @@ def test_windows_wheel_cuda(self):
             cuda=True,
             rocm=True,
             cpu=True,
+            xpu=False,
             reference_output_file="build_matrix_windows_wheel_cuda.json",
         )
 
@@ -97,6 +104,7 @@ def test_windows_conda_cuda(self):
             cuda=True,
             rocm=True,
             cpu=True,
+            xpu=True,
             reference_output_file="build_matrix_windows_conda_cuda.json",
         )
 
@@ -107,6 +115,7 @@ def test_linux_wheel_cuda_norocm(self):
             cuda=True,
             rocm=False,
             cpu=True,
+            xpu=False,
             reference_output_file="build_matrix_linux_wheel_cuda_norocm.json",
         )
 
@@ -117,8 +126,20 @@ def test_linux_wheel_cuda_rocm_nocpu(self):
             cuda=True,
             rocm=True,
             cpu=False,
+            xpu=False,
             reference_output_file="build_matrix_linux_wheel_nocpu.json",
         )
+
+    def test_linux_wheel_cuda_xpu_nocpu(self):
+        self.matrix_compare_helper(
+            package_type="wheel",
+            operating_system="linux",
+            cuda=True,
+            rocm=False,
+            cpu=False,
+            xpu=True,
+            reference_output_file="build_matrix_linux_wheel_xpu.json",
+        )
 
 
 if __name__ == "__main__":