Skip to content

Commit

Permalink
generate_binary_build_matrix - Return emply list for all conda packages
Browse files Browse the repository at this point in the history
  • Loading branch information
@atalman
atalman committed Nov 18, 2024
1 parent 661fe3c commit 2d7ad16
Show file tree
Hide file tree
Showing 4 changed files with 5 additions and 74 deletions.
73 changes: 2 additions & 71 deletions tools/scripts/generate_binary_build_matrix.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,6 @@
PYTHON_ARCHES = PYTHON_ARCHES_DICT[NIGHTLY]

# Container images
CONDA_CONTAINER_IMAGES: Dict[str, str]
LIBTORCH_CONTAINER_IMAGES: Dict[Tuple[str, str], str]
WHEEL_CONTAINER_IMAGES: Dict[str, str]

Expand All @@ -88,9 +87,6 @@
MACOS_M1_RUNNER = "macos-m1-stable"

PACKAGES_TO_INSTALL_WHL = "torch torchvision torchaudio"

PACKAGES_TO_INSTALL_CONDA = "pytorch torchvision torchaudio"
CONDA_INSTALL_BASE = f"conda install {PACKAGES_TO_INSTALL_CONDA}"
WHL_INSTALL_BASE = "pip3 install"
DOWNLOAD_URL_BASE = "https://download.pytorch.org"

Expand Down Expand Up @@ -137,7 +133,7 @@ def validation_runner(arch_type: str, os: str) -> str:

def initialize_globals(channel: str, build_python_only: bool) -> None:
global CURRENT_VERSION, CUDA_ARCHES, ROCM_ARCHES, PYTHON_ARCHES
global WHEEL_CONTAINER_IMAGES, CONDA_CONTAINER_IMAGES, LIBTORCH_CONTAINER_IMAGES
global WHEEL_CONTAINER_IMAGES, LIBTORCH_CONTAINER_IMAGES
if channel == TEST:
CURRENT_VERSION = CURRENT_CANDIDATE_VERSION
else:
Expand Down Expand Up @@ -165,13 +161,6 @@ def initialize_globals(channel: str, build_python_only: bool) -> None:
CPU_AARCH64: "pytorch/manylinux2_28_aarch64-builder:cpu-aarch64",
CUDA_AARCH64: "pytorch/manylinuxaarch64-builder:cuda12.4",
}
CONDA_CONTAINER_IMAGES = {
**{
gpu_arch: f"pytorch/conda-builder:cuda{gpu_arch}"
for gpu_arch in CUDA_ARCHES
},
CPU: "pytorch/conda-builder:cpu",
}
LIBTORCH_CONTAINER_IMAGES = {
**{
(gpu_arch, PRE_CXX11_ABI): f"pytorch/manylinux-builder:cuda{gpu_arch}"
Expand Down Expand Up @@ -209,23 +198,6 @@ def list_without(in_list: List[str], without: List[str]) -> List[str]:
return [item for item in in_list if item not in without]


def get_conda_install_command(
channel: str, gpu_arch_type: str, arch_version: str, os: str
) -> str:
pytorch_channel = "pytorch" if channel == RELEASE else f"pytorch-{channel}"
conda_channels = f"-c {pytorch_channel}"
conda_package_type = ""
if gpu_arch_type == CUDA:
conda_package_type = f"pytorch-cuda={arch_version}"
conda_channels = f"{conda_channels} -c nvidia"
elif os not in (MACOS_ARM64):
conda_package_type = "cpuonly"
else:
return f"conda install {pytorch_channel}::{PACKAGES_TO_INSTALL_CONDA} {conda_channels}"

return f"{CONDA_INSTALL_BASE} {conda_package_type} {conda_channels}"


def get_base_download_url_for_repo(
repo: str, channel: str, gpu_arch_type: str, desired_cuda: str
) -> str:
Expand Down Expand Up @@ -339,48 +311,7 @@ def generate_conda_matrix(
use_split_build: bool = False,
) -> List[Dict[str, str]]:
ret: List[Dict[str, str]] = []
python_versions = list(PYTHON_ARCHES)

arches = []

if with_cpu == ENABLE:
arches += [CPU]

if with_cuda == ENABLE:
if os == LINUX or os == WINDOWS:
arches += CUDA_ARCHES

if limit_pr_builds:
python_versions = [python_versions[0]]

for python_version in python_versions:
# We don't currently build conda packages for rocm
for arch_version in arches:
gpu_arch_type = arch_type(arch_version)
gpu_arch_version = "" if arch_version == CPU else arch_version

ret.append(
{
"python_version": python_version,
"gpu_arch_type": gpu_arch_type,
"gpu_arch_version": gpu_arch_version,
"desired_cuda": translate_desired_cuda(
gpu_arch_type, gpu_arch_version
),
"container_image": CONDA_CONTAINER_IMAGES[arch_version],
"package_type": "conda",
"build_name": f"conda-py{python_version}-{gpu_arch_type}{gpu_arch_version}".replace(
".", "_"
),
"validation_runner": validation_runner(gpu_arch_type, os),
"channel": channel,
"stable_version": CURRENT_VERSION,
"installation": get_conda_install_command(
channel, gpu_arch_type, arch_version, os
),
}
)

# return empty list. Conda builds are deprecated, see https://github.com/pytorch/pytorch/issues/138506
return ret


Expand Down
2 changes: 1 addition & 1 deletion tools/tests/assets/build_matrix_linux_conda_cuda.json
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"include": [{"python_version": "3.9", "gpu_arch_type": "cpu", "gpu_arch_version": "", "desired_cuda": "cpu", "container_image": "pytorch/conda-builder:cpu", "package_type": "conda", "build_name": "conda-py3_9-cpu", "validation_runner": "linux.2xlarge", "channel": "nightly", "stable_version": "2.5.1", "installation": "conda install pytorch torchvision torchaudio cpuonly -c pytorch-nightly"}, {"python_version": "3.9", "gpu_arch_type": "cuda", "gpu_arch_version": "11.8", "desired_cuda": "cu118", "container_image": "pytorch/conda-builder:cuda11.8", "package_type": "conda", "build_name": "conda-py3_9-cuda11_8", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "channel": "nightly", "stable_version": "2.5.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=11.8 -c pytorch-nightly -c nvidia"}, {"python_version": "3.9", "gpu_arch_type": "cuda", "gpu_arch_version": "12.4", "desired_cuda": "cu124", "container_image": "pytorch/conda-builder:cuda12.4", "package_type": "conda", "build_name": "conda-py3_9-cuda12_4", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "channel": "nightly", "stable_version": "2.5.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=12.4 -c pytorch-nightly -c nvidia"}, {"python_version": "3.9", "gpu_arch_type": "cuda", "gpu_arch_version": "12.6", "desired_cuda": "cu126", "container_image": "pytorch/conda-builder:cuda12.6", "package_type": "conda", "build_name": "conda-py3_9-cuda12_6", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "channel": "nightly", "stable_version": "2.5.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=12.6 -c pytorch-nightly -c nvidia"}, {"python_version": "3.10", "gpu_arch_type": "cpu", "gpu_arch_version": "", "desired_cuda": "cpu", "container_image": "pytorch/conda-builder:cpu", "package_type": "conda", "build_name": "conda-py3_10-cpu", "validation_runner": "linux.2xlarge", "channel": "nightly", "stable_version": "2.5.1", "installation": "conda install pytorch torchvision torchaudio cpuonly -c pytorch-nightly"}, {"python_version": "3.10", "gpu_arch_type": "cuda", "gpu_arch_version": "11.8", "desired_cuda": "cu118", "container_image": "pytorch/conda-builder:cuda11.8", "package_type": "conda", "build_name": "conda-py3_10-cuda11_8", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "channel": "nightly", "stable_version": "2.5.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=11.8 -c pytorch-nightly -c nvidia"}, {"python_version": "3.10", "gpu_arch_type": "cuda", "gpu_arch_version": "12.4", "desired_cuda": "cu124", "container_image": "pytorch/conda-builder:cuda12.4", "package_type": "conda", "build_name": "conda-py3_10-cuda12_4", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "channel": "nightly", "stable_version": "2.5.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=12.4 -c pytorch-nightly -c nvidia"}, {"python_version": "3.10", "gpu_arch_type": "cuda", "gpu_arch_version": "12.6", "desired_cuda": "cu126", "container_image": "pytorch/conda-builder:cuda12.6", "package_type": "conda", "build_name": "conda-py3_10-cuda12_6", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "channel": "nightly", "stable_version": "2.5.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=12.6 -c pytorch-nightly -c nvidia"}, {"python_version": "3.11", "gpu_arch_type": "cpu", "gpu_arch_version": "", "desired_cuda": "cpu", "container_image": "pytorch/conda-builder:cpu", "package_type": "conda", "build_name": "conda-py3_11-cpu", "validation_runner": "linux.2xlarge", "channel": "nightly", "stable_version": "2.5.1", "installation": "conda install pytorch torchvision torchaudio cpuonly -c pytorch-nightly"}, {"python_version": "3.11", "gpu_arch_type": "cuda", "gpu_arch_version": "11.8", "desired_cuda": "cu118", "container_image": "pytorch/conda-builder:cuda11.8", "package_type": "conda", "build_name": "conda-py3_11-cuda11_8", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "channel": "nightly", "stable_version": "2.5.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=11.8 -c pytorch-nightly -c nvidia"}, {"python_version": "3.11", "gpu_arch_type": "cuda", "gpu_arch_version": "12.4", "desired_cuda": "cu124", "container_image": "pytorch/conda-builder:cuda12.4", "package_type": "conda", "build_name": "conda-py3_11-cuda12_4", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "channel": "nightly", "stable_version": "2.5.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=12.4 -c pytorch-nightly -c nvidia"}, {"python_version": "3.11", "gpu_arch_type": "cuda", "gpu_arch_version": "12.6", "desired_cuda": "cu126", "container_image": "pytorch/conda-builder:cuda12.6", "package_type": "conda", "build_name": "conda-py3_11-cuda12_6", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "channel": "nightly", "stable_version": "2.5.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=12.6 -c pytorch-nightly -c nvidia"}, {"python_version": "3.12", "gpu_arch_type": "cpu", "gpu_arch_version": "", "desired_cuda": "cpu", "container_image": "pytorch/conda-builder:cpu", "package_type": "conda", "build_name": "conda-py3_12-cpu", "validation_runner": "linux.2xlarge", "channel": "nightly", "stable_version": "2.5.1", "installation": "conda install pytorch torchvision torchaudio cpuonly -c pytorch-nightly"}, {"python_version": "3.12", "gpu_arch_type": "cuda", "gpu_arch_version": "11.8", "desired_cuda": "cu118", "container_image": "pytorch/conda-builder:cuda11.8", "package_type": "conda", "build_name": "conda-py3_12-cuda11_8", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "channel": "nightly", "stable_version": "2.5.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=11.8 -c pytorch-nightly -c nvidia"}, {"python_version": "3.12", "gpu_arch_type": "cuda", "gpu_arch_version": "12.4", "desired_cuda": "cu124", "container_image": "pytorch/conda-builder:cuda12.4", "package_type": "conda", "build_name": "conda-py3_12-cuda12_4", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "channel": "nightly", "stable_version": "2.5.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=12.4 -c pytorch-nightly -c nvidia"}, {"python_version": "3.12", "gpu_arch_type": "cuda", "gpu_arch_version": "12.6", "desired_cuda": "cu126", "container_image": "pytorch/conda-builder:cuda12.6", "package_type": "conda", "build_name": "conda-py3_12-cuda12_6", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "channel": "nightly", "stable_version": "2.5.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=12.6 -c pytorch-nightly -c nvidia"}]}
{"include": []}
2 changes: 1 addition & 1 deletion tools/tests/assets/build_matrix_macos_conda.json
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"include": [{"python_version": "3.9", "gpu_arch_type": "cpu", "gpu_arch_version": "", "desired_cuda": "cpu", "container_image": "pytorch/conda-builder:cpu", "package_type": "conda", "build_name": "conda-py3_9-cpu", "validation_runner": "linux.2xlarge", "channel": "nightly", "stable_version": "2.5.1", "installation": "conda install pytorch-nightly::pytorch torchvision torchaudio -c pytorch-nightly"}, {"python_version": "3.10", "gpu_arch_type": "cpu", "gpu_arch_version": "", "desired_cuda": "cpu", "container_image": "pytorch/conda-builder:cpu", "package_type": "conda", "build_name": "conda-py3_10-cpu", "validation_runner": "linux.2xlarge", "channel": "nightly", "stable_version": "2.5.1", "installation": "conda install pytorch-nightly::pytorch torchvision torchaudio -c pytorch-nightly"}, {"python_version": "3.11", "gpu_arch_type": "cpu", "gpu_arch_version": "", "desired_cuda": "cpu", "container_image": "pytorch/conda-builder:cpu", "package_type": "conda", "build_name": "conda-py3_11-cpu", "validation_runner": "linux.2xlarge", "channel": "nightly", "stable_version": "2.5.1", "installation": "conda install pytorch-nightly::pytorch torchvision torchaudio -c pytorch-nightly"}, {"python_version": "3.12", "gpu_arch_type": "cpu", "gpu_arch_version": "", "desired_cuda": "cpu", "container_image": "pytorch/conda-builder:cpu", "package_type": "conda", "build_name": "conda-py3_12-cpu", "validation_runner": "linux.2xlarge", "channel": "nightly", "stable_version": "2.5.1", "installation": "conda install pytorch-nightly::pytorch torchvision torchaudio -c pytorch-nightly"}]}
{"include": []}
2 changes: 1 addition & 1 deletion tools/tests/assets/build_matrix_windows_conda_cuda.json
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"include": [{"python_version": "3.9", "gpu_arch_type": "cpu", "gpu_arch_version": "", "desired_cuda": "cpu", "container_image": "pytorch/conda-builder:cpu", "package_type": "conda", "build_name": "conda-py3_9-cpu", "validation_runner": "windows.4xlarge", "channel": "nightly", "stable_version": "2.5.1", "installation": "conda install pytorch torchvision torchaudio cpuonly -c pytorch-nightly"}, {"python_version": "3.9", "gpu_arch_type": "cuda", "gpu_arch_version": "11.8", "desired_cuda": "cu118", "container_image": "pytorch/conda-builder:cuda11.8", "package_type": "conda", "build_name": "conda-py3_9-cuda11_8", "validation_runner": "windows.g4dn.xlarge", "channel": "nightly", "stable_version": "2.5.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=11.8 -c pytorch-nightly -c nvidia"}, {"python_version": "3.9", "gpu_arch_type": "cuda", "gpu_arch_version": "12.4", "desired_cuda": "cu124", "container_image": "pytorch/conda-builder:cuda12.4", "package_type": "conda", "build_name": "conda-py3_9-cuda12_4", "validation_runner": "windows.g4dn.xlarge", "channel": "nightly", "stable_version": "2.5.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=12.4 -c pytorch-nightly -c nvidia"}, {"python_version": "3.9", "gpu_arch_type": "cuda", "gpu_arch_version": "12.6", "desired_cuda": "cu126", "container_image": "pytorch/conda-builder:cuda12.6", "package_type": "conda", "build_name": "conda-py3_9-cuda12_6", "validation_runner": "windows.g4dn.xlarge", "channel": "nightly", "stable_version": "2.5.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=12.6 -c pytorch-nightly -c nvidia"}, {"python_version": "3.10", "gpu_arch_type": "cpu", "gpu_arch_version": "", "desired_cuda": "cpu", "container_image": "pytorch/conda-builder:cpu", "package_type": "conda", "build_name": "conda-py3_10-cpu", "validation_runner": "windows.4xlarge", "channel": "nightly", "stable_version": "2.5.1", "installation": "conda install pytorch torchvision torchaudio cpuonly -c pytorch-nightly"}, {"python_version": "3.10", "gpu_arch_type": "cuda", "gpu_arch_version": "11.8", "desired_cuda": "cu118", "container_image": "pytorch/conda-builder:cuda11.8", "package_type": "conda", "build_name": "conda-py3_10-cuda11_8", "validation_runner": "windows.g4dn.xlarge", "channel": "nightly", "stable_version": "2.5.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=11.8 -c pytorch-nightly -c nvidia"}, {"python_version": "3.10", "gpu_arch_type": "cuda", "gpu_arch_version": "12.4", "desired_cuda": "cu124", "container_image": "pytorch/conda-builder:cuda12.4", "package_type": "conda", "build_name": "conda-py3_10-cuda12_4", "validation_runner": "windows.g4dn.xlarge", "channel": "nightly", "stable_version": "2.5.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=12.4 -c pytorch-nightly -c nvidia"}, {"python_version": "3.10", "gpu_arch_type": "cuda", "gpu_arch_version": "12.6", "desired_cuda": "cu126", "container_image": "pytorch/conda-builder:cuda12.6", "package_type": "conda", "build_name": "conda-py3_10-cuda12_6", "validation_runner": "windows.g4dn.xlarge", "channel": "nightly", "stable_version": "2.5.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=12.6 -c pytorch-nightly -c nvidia"}, {"python_version": "3.11", "gpu_arch_type": "cpu", "gpu_arch_version": "", "desired_cuda": "cpu", "container_image": "pytorch/conda-builder:cpu", "package_type": "conda", "build_name": "conda-py3_11-cpu", "validation_runner": "windows.4xlarge", "channel": "nightly", "stable_version": "2.5.1", "installation": "conda install pytorch torchvision torchaudio cpuonly -c pytorch-nightly"}, {"python_version": "3.11", "gpu_arch_type": "cuda", "gpu_arch_version": "11.8", "desired_cuda": "cu118", "container_image": "pytorch/conda-builder:cuda11.8", "package_type": "conda", "build_name": "conda-py3_11-cuda11_8", "validation_runner": "windows.g4dn.xlarge", "channel": "nightly", "stable_version": "2.5.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=11.8 -c pytorch-nightly -c nvidia"}, {"python_version": "3.11", "gpu_arch_type": "cuda", "gpu_arch_version": "12.4", "desired_cuda": "cu124", "container_image": "pytorch/conda-builder:cuda12.4", "package_type": "conda", "build_name": "conda-py3_11-cuda12_4", "validation_runner": "windows.g4dn.xlarge", "channel": "nightly", "stable_version": "2.5.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=12.4 -c pytorch-nightly -c nvidia"}, {"python_version": "3.11", "gpu_arch_type": "cuda", "gpu_arch_version": "12.6", "desired_cuda": "cu126", "container_image": "pytorch/conda-builder:cuda12.6", "package_type": "conda", "build_name": "conda-py3_11-cuda12_6", "validation_runner": "windows.g4dn.xlarge", "channel": "nightly", "stable_version": "2.5.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=12.6 -c pytorch-nightly -c nvidia"}, {"python_version": "3.12", "gpu_arch_type": "cpu", "gpu_arch_version": "", "desired_cuda": "cpu", "container_image": "pytorch/conda-builder:cpu", "package_type": "conda", "build_name": "conda-py3_12-cpu", "validation_runner": "windows.4xlarge", "channel": "nightly", "stable_version": "2.5.1", "installation": "conda install pytorch torchvision torchaudio cpuonly -c pytorch-nightly"}, {"python_version": "3.12", "gpu_arch_type": "cuda", "gpu_arch_version": "11.8", "desired_cuda": "cu118", "container_image": "pytorch/conda-builder:cuda11.8", "package_type": "conda", "build_name": "conda-py3_12-cuda11_8", "validation_runner": "windows.g4dn.xlarge", "channel": "nightly", "stable_version": "2.5.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=11.8 -c pytorch-nightly -c nvidia"}, {"python_version": "3.12", "gpu_arch_type": "cuda", "gpu_arch_version": "12.4", "desired_cuda": "cu124", "container_image": "pytorch/conda-builder:cuda12.4", "package_type": "conda", "build_name": "conda-py3_12-cuda12_4", "validation_runner": "windows.g4dn.xlarge", "channel": "nightly", "stable_version": "2.5.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=12.4 -c pytorch-nightly -c nvidia"}, {"python_version": "3.12", "gpu_arch_type": "cuda", "gpu_arch_version": "12.6", "desired_cuda": "cu126", "container_image": "pytorch/conda-builder:cuda12.6", "package_type": "conda", "build_name": "conda-py3_12-cuda12_6", "validation_runner": "windows.g4dn.xlarge", "channel": "nightly", "stable_version": "2.5.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=12.6 -c pytorch-nightly -c nvidia"}]}
{"include": []}

0 comments on commit 2d7ad16

Please sign in to comment.