From 2d7ad16410476c73ada9d9c5fe9872f25487afd3 Mon Sep 17 00:00:00 2001 From: atalman Date: Mon, 18 Nov 2024 12:29:44 -0800 Subject: [PATCH] generate_binary_build_matrix - Return emply list for all conda packages --- tools/scripts/generate_binary_build_matrix.py | 73 +------------------ .../assets/build_matrix_linux_conda_cuda.json | 2 +- .../assets/build_matrix_macos_conda.json | 2 +- .../build_matrix_windows_conda_cuda.json | 2 +- 4 files changed, 5 insertions(+), 74 deletions(-) diff --git a/tools/scripts/generate_binary_build_matrix.py b/tools/scripts/generate_binary_build_matrix.py index ab7cc214c7..6ed4834438 100644 --- a/tools/scripts/generate_binary_build_matrix.py +++ b/tools/scripts/generate_binary_build_matrix.py @@ -75,7 +75,6 @@ PYTHON_ARCHES = PYTHON_ARCHES_DICT[NIGHTLY] # Container images -CONDA_CONTAINER_IMAGES: Dict[str, str] LIBTORCH_CONTAINER_IMAGES: Dict[Tuple[str, str], str] WHEEL_CONTAINER_IMAGES: Dict[str, str] @@ -88,9 +87,6 @@ MACOS_M1_RUNNER = "macos-m1-stable" PACKAGES_TO_INSTALL_WHL = "torch torchvision torchaudio" - -PACKAGES_TO_INSTALL_CONDA = "pytorch torchvision torchaudio" -CONDA_INSTALL_BASE = f"conda install {PACKAGES_TO_INSTALL_CONDA}" WHL_INSTALL_BASE = "pip3 install" DOWNLOAD_URL_BASE = "https://download.pytorch.org" @@ -137,7 +133,7 @@ def validation_runner(arch_type: str, os: str) -> str: def initialize_globals(channel: str, build_python_only: bool) -> None: global CURRENT_VERSION, CUDA_ARCHES, ROCM_ARCHES, PYTHON_ARCHES - global WHEEL_CONTAINER_IMAGES, CONDA_CONTAINER_IMAGES, LIBTORCH_CONTAINER_IMAGES + global WHEEL_CONTAINER_IMAGES, LIBTORCH_CONTAINER_IMAGES if channel == TEST: CURRENT_VERSION = CURRENT_CANDIDATE_VERSION else: @@ -165,13 +161,6 @@ def initialize_globals(channel: str, build_python_only: bool) -> None: CPU_AARCH64: "pytorch/manylinux2_28_aarch64-builder:cpu-aarch64", CUDA_AARCH64: "pytorch/manylinuxaarch64-builder:cuda12.4", } - CONDA_CONTAINER_IMAGES = { - **{ - gpu_arch: f"pytorch/conda-builder:cuda{gpu_arch}" - for gpu_arch in CUDA_ARCHES - }, - CPU: "pytorch/conda-builder:cpu", - } LIBTORCH_CONTAINER_IMAGES = { **{ (gpu_arch, PRE_CXX11_ABI): f"pytorch/manylinux-builder:cuda{gpu_arch}" @@ -209,23 +198,6 @@ def list_without(in_list: List[str], without: List[str]) -> List[str]: return [item for item in in_list if item not in without] -def get_conda_install_command( - channel: str, gpu_arch_type: str, arch_version: str, os: str -) -> str: - pytorch_channel = "pytorch" if channel == RELEASE else f"pytorch-{channel}" - conda_channels = f"-c {pytorch_channel}" - conda_package_type = "" - if gpu_arch_type == CUDA: - conda_package_type = f"pytorch-cuda={arch_version}" - conda_channels = f"{conda_channels} -c nvidia" - elif os not in (MACOS_ARM64): - conda_package_type = "cpuonly" - else: - return f"conda install {pytorch_channel}::{PACKAGES_TO_INSTALL_CONDA} {conda_channels}" - - return f"{CONDA_INSTALL_BASE} {conda_package_type} {conda_channels}" - - def get_base_download_url_for_repo( repo: str, channel: str, gpu_arch_type: str, desired_cuda: str ) -> str: @@ -339,48 +311,7 @@ def generate_conda_matrix( use_split_build: bool = False, ) -> List[Dict[str, str]]: ret: List[Dict[str, str]] = [] - python_versions = list(PYTHON_ARCHES) - - arches = [] - - if with_cpu == ENABLE: - arches += [CPU] - - if with_cuda == ENABLE: - if os == LINUX or os == WINDOWS: - arches += CUDA_ARCHES - - if limit_pr_builds: - python_versions = [python_versions[0]] - - for python_version in python_versions: - # We don't currently build conda packages for rocm - for arch_version in arches: - gpu_arch_type = arch_type(arch_version) - gpu_arch_version = "" if arch_version == CPU else arch_version - - ret.append( - { - "python_version": python_version, - "gpu_arch_type": gpu_arch_type, - "gpu_arch_version": gpu_arch_version, - "desired_cuda": translate_desired_cuda( - gpu_arch_type, gpu_arch_version - ), - "container_image": CONDA_CONTAINER_IMAGES[arch_version], - "package_type": "conda", - "build_name": f"conda-py{python_version}-{gpu_arch_type}{gpu_arch_version}".replace( - ".", "_" - ), - "validation_runner": validation_runner(gpu_arch_type, os), - "channel": channel, - "stable_version": CURRENT_VERSION, - "installation": get_conda_install_command( - channel, gpu_arch_type, arch_version, os - ), - } - ) - + # return empty list. Conda builds are deprecated, see https://github.com/pytorch/pytorch/issues/138506 return ret diff --git a/tools/tests/assets/build_matrix_linux_conda_cuda.json b/tools/tests/assets/build_matrix_linux_conda_cuda.json index 2d5834654e..efafe1abef 100644 --- a/tools/tests/assets/build_matrix_linux_conda_cuda.json +++ b/tools/tests/assets/build_matrix_linux_conda_cuda.json @@ -1 +1 @@ -{"include": [{"python_version": "3.9", "gpu_arch_type": "cpu", "gpu_arch_version": "", "desired_cuda": "cpu", "container_image": "pytorch/conda-builder:cpu", "package_type": "conda", "build_name": "conda-py3_9-cpu", "validation_runner": "linux.2xlarge", "channel": "nightly", "stable_version": "2.5.1", "installation": "conda install pytorch torchvision torchaudio cpuonly -c pytorch-nightly"}, {"python_version": "3.9", "gpu_arch_type": "cuda", "gpu_arch_version": "11.8", "desired_cuda": "cu118", "container_image": "pytorch/conda-builder:cuda11.8", "package_type": "conda", "build_name": "conda-py3_9-cuda11_8", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "channel": "nightly", "stable_version": "2.5.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=11.8 -c pytorch-nightly -c nvidia"}, {"python_version": "3.9", "gpu_arch_type": "cuda", "gpu_arch_version": "12.4", "desired_cuda": "cu124", "container_image": "pytorch/conda-builder:cuda12.4", "package_type": "conda", "build_name": "conda-py3_9-cuda12_4", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "channel": "nightly", "stable_version": "2.5.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=12.4 -c pytorch-nightly -c nvidia"}, {"python_version": "3.9", "gpu_arch_type": "cuda", "gpu_arch_version": "12.6", "desired_cuda": "cu126", "container_image": "pytorch/conda-builder:cuda12.6", "package_type": "conda", "build_name": "conda-py3_9-cuda12_6", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "channel": "nightly", "stable_version": "2.5.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=12.6 -c pytorch-nightly -c nvidia"}, {"python_version": "3.10", "gpu_arch_type": "cpu", "gpu_arch_version": "", "desired_cuda": "cpu", "container_image": "pytorch/conda-builder:cpu", "package_type": "conda", "build_name": "conda-py3_10-cpu", "validation_runner": "linux.2xlarge", "channel": "nightly", "stable_version": "2.5.1", "installation": "conda install pytorch torchvision torchaudio cpuonly -c pytorch-nightly"}, {"python_version": "3.10", "gpu_arch_type": "cuda", "gpu_arch_version": "11.8", "desired_cuda": "cu118", "container_image": "pytorch/conda-builder:cuda11.8", "package_type": "conda", "build_name": "conda-py3_10-cuda11_8", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "channel": "nightly", "stable_version": "2.5.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=11.8 -c pytorch-nightly -c nvidia"}, {"python_version": "3.10", "gpu_arch_type": "cuda", "gpu_arch_version": "12.4", "desired_cuda": "cu124", "container_image": "pytorch/conda-builder:cuda12.4", "package_type": "conda", "build_name": "conda-py3_10-cuda12_4", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "channel": "nightly", "stable_version": "2.5.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=12.4 -c pytorch-nightly -c nvidia"}, {"python_version": "3.10", "gpu_arch_type": "cuda", "gpu_arch_version": "12.6", "desired_cuda": "cu126", "container_image": "pytorch/conda-builder:cuda12.6", "package_type": "conda", "build_name": "conda-py3_10-cuda12_6", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "channel": "nightly", "stable_version": "2.5.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=12.6 -c pytorch-nightly -c nvidia"}, {"python_version": "3.11", "gpu_arch_type": "cpu", "gpu_arch_version": "", "desired_cuda": "cpu", "container_image": "pytorch/conda-builder:cpu", "package_type": "conda", "build_name": "conda-py3_11-cpu", "validation_runner": "linux.2xlarge", "channel": "nightly", "stable_version": "2.5.1", "installation": "conda install pytorch torchvision torchaudio cpuonly -c pytorch-nightly"}, {"python_version": "3.11", "gpu_arch_type": "cuda", "gpu_arch_version": "11.8", "desired_cuda": "cu118", "container_image": "pytorch/conda-builder:cuda11.8", "package_type": "conda", "build_name": "conda-py3_11-cuda11_8", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "channel": "nightly", "stable_version": "2.5.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=11.8 -c pytorch-nightly -c nvidia"}, {"python_version": "3.11", "gpu_arch_type": "cuda", "gpu_arch_version": "12.4", "desired_cuda": "cu124", "container_image": "pytorch/conda-builder:cuda12.4", "package_type": "conda", "build_name": "conda-py3_11-cuda12_4", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "channel": "nightly", "stable_version": "2.5.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=12.4 -c pytorch-nightly -c nvidia"}, {"python_version": "3.11", "gpu_arch_type": "cuda", "gpu_arch_version": "12.6", "desired_cuda": "cu126", "container_image": "pytorch/conda-builder:cuda12.6", "package_type": "conda", "build_name": "conda-py3_11-cuda12_6", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "channel": "nightly", "stable_version": "2.5.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=12.6 -c pytorch-nightly -c nvidia"}, {"python_version": "3.12", "gpu_arch_type": "cpu", "gpu_arch_version": "", "desired_cuda": "cpu", "container_image": "pytorch/conda-builder:cpu", "package_type": "conda", "build_name": "conda-py3_12-cpu", "validation_runner": "linux.2xlarge", "channel": "nightly", "stable_version": "2.5.1", "installation": "conda install pytorch torchvision torchaudio cpuonly -c pytorch-nightly"}, {"python_version": "3.12", "gpu_arch_type": "cuda", "gpu_arch_version": "11.8", "desired_cuda": "cu118", "container_image": "pytorch/conda-builder:cuda11.8", "package_type": "conda", "build_name": "conda-py3_12-cuda11_8", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "channel": "nightly", "stable_version": "2.5.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=11.8 -c pytorch-nightly -c nvidia"}, {"python_version": "3.12", "gpu_arch_type": "cuda", "gpu_arch_version": "12.4", "desired_cuda": "cu124", "container_image": "pytorch/conda-builder:cuda12.4", "package_type": "conda", "build_name": "conda-py3_12-cuda12_4", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "channel": "nightly", "stable_version": "2.5.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=12.4 -c pytorch-nightly -c nvidia"}, {"python_version": "3.12", "gpu_arch_type": "cuda", "gpu_arch_version": "12.6", "desired_cuda": "cu126", "container_image": "pytorch/conda-builder:cuda12.6", "package_type": "conda", "build_name": "conda-py3_12-cuda12_6", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "channel": "nightly", "stable_version": "2.5.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=12.6 -c pytorch-nightly -c nvidia"}]} +{"include": []} diff --git a/tools/tests/assets/build_matrix_macos_conda.json b/tools/tests/assets/build_matrix_macos_conda.json index ac01101ba3..efafe1abef 100644 --- a/tools/tests/assets/build_matrix_macos_conda.json +++ b/tools/tests/assets/build_matrix_macos_conda.json @@ -1 +1 @@ -{"include": [{"python_version": "3.9", "gpu_arch_type": "cpu", "gpu_arch_version": "", "desired_cuda": "cpu", "container_image": "pytorch/conda-builder:cpu", "package_type": "conda", "build_name": "conda-py3_9-cpu", "validation_runner": "linux.2xlarge", "channel": "nightly", "stable_version": "2.5.1", "installation": "conda install pytorch-nightly::pytorch torchvision torchaudio -c pytorch-nightly"}, {"python_version": "3.10", "gpu_arch_type": "cpu", "gpu_arch_version": "", "desired_cuda": "cpu", "container_image": "pytorch/conda-builder:cpu", "package_type": "conda", "build_name": "conda-py3_10-cpu", "validation_runner": "linux.2xlarge", "channel": "nightly", "stable_version": "2.5.1", "installation": "conda install pytorch-nightly::pytorch torchvision torchaudio -c pytorch-nightly"}, {"python_version": "3.11", "gpu_arch_type": "cpu", "gpu_arch_version": "", "desired_cuda": "cpu", "container_image": "pytorch/conda-builder:cpu", "package_type": "conda", "build_name": "conda-py3_11-cpu", "validation_runner": "linux.2xlarge", "channel": "nightly", "stable_version": "2.5.1", "installation": "conda install pytorch-nightly::pytorch torchvision torchaudio -c pytorch-nightly"}, {"python_version": "3.12", "gpu_arch_type": "cpu", "gpu_arch_version": "", "desired_cuda": "cpu", "container_image": "pytorch/conda-builder:cpu", "package_type": "conda", "build_name": "conda-py3_12-cpu", "validation_runner": "linux.2xlarge", "channel": "nightly", "stable_version": "2.5.1", "installation": "conda install pytorch-nightly::pytorch torchvision torchaudio -c pytorch-nightly"}]} +{"include": []} diff --git a/tools/tests/assets/build_matrix_windows_conda_cuda.json b/tools/tests/assets/build_matrix_windows_conda_cuda.json index f33a420edb..efafe1abef 100644 --- a/tools/tests/assets/build_matrix_windows_conda_cuda.json +++ b/tools/tests/assets/build_matrix_windows_conda_cuda.json @@ -1 +1 @@ -{"include": [{"python_version": "3.9", "gpu_arch_type": "cpu", "gpu_arch_version": "", "desired_cuda": "cpu", "container_image": "pytorch/conda-builder:cpu", "package_type": "conda", "build_name": "conda-py3_9-cpu", "validation_runner": "windows.4xlarge", "channel": "nightly", "stable_version": "2.5.1", "installation": "conda install pytorch torchvision torchaudio cpuonly -c pytorch-nightly"}, {"python_version": "3.9", "gpu_arch_type": "cuda", "gpu_arch_version": "11.8", "desired_cuda": "cu118", "container_image": "pytorch/conda-builder:cuda11.8", "package_type": "conda", "build_name": "conda-py3_9-cuda11_8", "validation_runner": "windows.g4dn.xlarge", "channel": "nightly", "stable_version": "2.5.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=11.8 -c pytorch-nightly -c nvidia"}, {"python_version": "3.9", "gpu_arch_type": "cuda", "gpu_arch_version": "12.4", "desired_cuda": "cu124", "container_image": "pytorch/conda-builder:cuda12.4", "package_type": "conda", "build_name": "conda-py3_9-cuda12_4", "validation_runner": "windows.g4dn.xlarge", "channel": "nightly", "stable_version": "2.5.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=12.4 -c pytorch-nightly -c nvidia"}, {"python_version": "3.9", "gpu_arch_type": "cuda", "gpu_arch_version": "12.6", "desired_cuda": "cu126", "container_image": "pytorch/conda-builder:cuda12.6", "package_type": "conda", "build_name": "conda-py3_9-cuda12_6", "validation_runner": "windows.g4dn.xlarge", "channel": "nightly", "stable_version": "2.5.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=12.6 -c pytorch-nightly -c nvidia"}, {"python_version": "3.10", "gpu_arch_type": "cpu", "gpu_arch_version": "", "desired_cuda": "cpu", "container_image": "pytorch/conda-builder:cpu", "package_type": "conda", "build_name": "conda-py3_10-cpu", "validation_runner": "windows.4xlarge", "channel": "nightly", "stable_version": "2.5.1", "installation": "conda install pytorch torchvision torchaudio cpuonly -c pytorch-nightly"}, {"python_version": "3.10", "gpu_arch_type": "cuda", "gpu_arch_version": "11.8", "desired_cuda": "cu118", "container_image": "pytorch/conda-builder:cuda11.8", "package_type": "conda", "build_name": "conda-py3_10-cuda11_8", "validation_runner": "windows.g4dn.xlarge", "channel": "nightly", "stable_version": "2.5.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=11.8 -c pytorch-nightly -c nvidia"}, {"python_version": "3.10", "gpu_arch_type": "cuda", "gpu_arch_version": "12.4", "desired_cuda": "cu124", "container_image": "pytorch/conda-builder:cuda12.4", "package_type": "conda", "build_name": "conda-py3_10-cuda12_4", "validation_runner": "windows.g4dn.xlarge", "channel": "nightly", "stable_version": "2.5.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=12.4 -c pytorch-nightly -c nvidia"}, {"python_version": "3.10", "gpu_arch_type": "cuda", "gpu_arch_version": "12.6", "desired_cuda": "cu126", "container_image": "pytorch/conda-builder:cuda12.6", "package_type": "conda", "build_name": "conda-py3_10-cuda12_6", "validation_runner": "windows.g4dn.xlarge", "channel": "nightly", "stable_version": "2.5.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=12.6 -c pytorch-nightly -c nvidia"}, {"python_version": "3.11", "gpu_arch_type": "cpu", "gpu_arch_version": "", "desired_cuda": "cpu", "container_image": "pytorch/conda-builder:cpu", "package_type": "conda", "build_name": "conda-py3_11-cpu", "validation_runner": "windows.4xlarge", "channel": "nightly", "stable_version": "2.5.1", "installation": "conda install pytorch torchvision torchaudio cpuonly -c pytorch-nightly"}, {"python_version": "3.11", "gpu_arch_type": "cuda", "gpu_arch_version": "11.8", "desired_cuda": "cu118", "container_image": "pytorch/conda-builder:cuda11.8", "package_type": "conda", "build_name": "conda-py3_11-cuda11_8", "validation_runner": "windows.g4dn.xlarge", "channel": "nightly", "stable_version": "2.5.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=11.8 -c pytorch-nightly -c nvidia"}, {"python_version": "3.11", "gpu_arch_type": "cuda", "gpu_arch_version": "12.4", "desired_cuda": "cu124", "container_image": "pytorch/conda-builder:cuda12.4", "package_type": "conda", "build_name": "conda-py3_11-cuda12_4", "validation_runner": "windows.g4dn.xlarge", "channel": "nightly", "stable_version": "2.5.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=12.4 -c pytorch-nightly -c nvidia"}, {"python_version": "3.11", "gpu_arch_type": "cuda", "gpu_arch_version": "12.6", "desired_cuda": "cu126", "container_image": "pytorch/conda-builder:cuda12.6", "package_type": "conda", "build_name": "conda-py3_11-cuda12_6", "validation_runner": "windows.g4dn.xlarge", "channel": "nightly", "stable_version": "2.5.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=12.6 -c pytorch-nightly -c nvidia"}, {"python_version": "3.12", "gpu_arch_type": "cpu", "gpu_arch_version": "", "desired_cuda": "cpu", "container_image": "pytorch/conda-builder:cpu", "package_type": "conda", "build_name": "conda-py3_12-cpu", "validation_runner": "windows.4xlarge", "channel": "nightly", "stable_version": "2.5.1", "installation": "conda install pytorch torchvision torchaudio cpuonly -c pytorch-nightly"}, {"python_version": "3.12", "gpu_arch_type": "cuda", "gpu_arch_version": "11.8", "desired_cuda": "cu118", "container_image": "pytorch/conda-builder:cuda11.8", "package_type": "conda", "build_name": "conda-py3_12-cuda11_8", "validation_runner": "windows.g4dn.xlarge", "channel": "nightly", "stable_version": "2.5.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=11.8 -c pytorch-nightly -c nvidia"}, {"python_version": "3.12", "gpu_arch_type": "cuda", "gpu_arch_version": "12.4", "desired_cuda": "cu124", "container_image": "pytorch/conda-builder:cuda12.4", "package_type": "conda", "build_name": "conda-py3_12-cuda12_4", "validation_runner": "windows.g4dn.xlarge", "channel": "nightly", "stable_version": "2.5.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=12.4 -c pytorch-nightly -c nvidia"}, {"python_version": "3.12", "gpu_arch_type": "cuda", "gpu_arch_version": "12.6", "desired_cuda": "cu126", "container_image": "pytorch/conda-builder:cuda12.6", "package_type": "conda", "build_name": "conda-py3_12-cuda12_6", "validation_runner": "windows.g4dn.xlarge", "channel": "nightly", "stable_version": "2.5.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=12.6 -c pytorch-nightly -c nvidia"}]} +{"include": []}