Skip to content

Commit

Permalink
Add docker image to docker release matrix (#5081)
Browse files Browse the repository at this point in the history 
Automate docker image validation. Add docker image to docker release
matrix.

Test:
```
(.venv) [[email protected] /data/users/atalman/test-infra/tools/scripts (generate_docker_matrix)]$ python generate_docker_release_matrix.py --channel release
{"include": [{"cuda": "11.8", "cuda_full_version": "11.8.0", "cudnn_version": "8", "image_type": "runtime", "docker": "ghcr.io/pytorch/pytorch:2.2.2-cuda11.8-cudnn8-runtime", "platform": "linux/arm64,linux/amd64"}, {"cuda": "11.8", "cuda_full_version": "11.8.0", "cudnn_version": "8", "image_type": "devel", "docker": "ghcr.io/pytorch/pytorch:2.2.2-cuda11.8-cudnn8-devel", "platform": "linux/arm64,linux/amd64"}, {"cuda": "12.1", "cuda_full_version": "12.1.1", "cudnn_version": "8", "image_type": "runtime", "docker": "ghcr.io/pytorch/pytorch:2.2.2-cuda12.1-cudnn8-runtime", "platform": "linux/arm64,linux/amd64"}, {"cuda": "12.1", "cuda_full_version": "12.1.1", "cudnn_version": "8", "image_type": "devel", "docker": "ghcr.io/pytorch/pytorch:2.2.2-cuda12.1-cudnn8-devel", "platform": "linux/arm64,linux/amd64"}]}
(.venv) [[email protected] /data/users/atalman/test-infra/tools/scripts (generate_docker_matrix)]$ python generate_docker_release_matrix.py --channel release
{"include": [{"cuda": "11.8", "cuda_full_version": "11.8.0", "cudnn_version": "8", "image_type": "runtime", "docker": "ghcr.io/pytorch/pytorch:2.2.2-cuda11.8-cudnn8-runtime", "platform": "linux/arm64,linux/amd64"}, {"cuda": "11.8", "cuda_full_version": "11.8.0", "cudnn_version": "8", "image_type": "devel", "docker": "ghcr.io/pytorch/pytorch:2.2.2-cuda11.8-cudnn8-devel", "platform": "linux/arm64,linux/amd64"}, {"cuda": "12.1", "cuda_full_version": "12.1.1", "cudnn_version": "8", "image_type": "runtime", "docker": "ghcr.io/pytorch/pytorch:2.2.2-cuda12.1-cudnn8-runtime", "platform": "linux/arm64,linux/amd64"}, {"cuda": "12.1", "cuda_full_version": "12.1.1", "cudnn_version": "8", "image_type": "devel", "docker": "ghcr.io/pytorch/pytorch:2.2.2-cuda12.1-cudnn8-devel", "platform": "linux/arm64,linux/amd64"}]}
(.venv) [[email protected] /data/users/atalman/test-infra/tools/scripts (generate_docker_matrix)]$ python generate_docker_release_matrix.py --channel test
{"include": [{"cuda": "11.8", "cuda_full_version": "11.8.0", "cudnn_version": "8", "image_type": "runtime", "docker": "ghcr.io/pytorch/pytorch-test:2.3.0-cuda11.8-cudnn8-runtime", "platform": "linux/arm64,linux/amd64"}, {"cuda": "11.8", "cuda_full_version": "11.8.0", "cudnn_version": "8", "image_type": "devel", "docker": "ghcr.io/pytorch/pytorch-test:2.3.0-cuda11.8-cudnn8-devel", "platform": "linux/arm64,linux/amd64"}, {"cuda": "12.1", "cuda_full_version": "12.1.1", "cudnn_version": "8", "image_type": "runtime", "docker": "ghcr.io/pytorch/pytorch-test:2.3.0-cuda12.1-cudnn8-runtime", "platform": "linux/arm64,linux/amd64"}, {"cuda": "12.1", "cuda_full_version": "12.1.1", "cudnn_version": "8", "image_type": "devel", "docker": "ghcr.io/pytorch/pytorch-test:2.3.0-cuda12.1-cudnn8-devel", "platform": "linux/arm64,linux/amd64"}]}
(.venv) [[email protected] /data/users/atalman/test-infra/tools/scripts (generate_docker_matrix)]$ python generate_docker_release_matrix.py --channel nightly
{"include": [{"cuda": "11.8", "cuda_full_version": "11.8.0", "cudnn_version": "8", "image_type": "runtime", "docker": "ghcr.io/pytorch/pytorch-nightly:2.4.0.dev20240410-cuda11.8-cudnn8-runtime", "platform": "linux/arm64,linux/amd64"}, {"cuda": "11.8", "cuda_full_version": "11.8.0", "cudnn_version": "8", "image_type": "devel", "docker": "ghcr.io/pytorch/pytorch-nightly:2.4.0.dev20240410-cuda11.8-cudnn8-devel", "platform": "linux/arm64,linux/amd64"}, {"cuda": "12.1", "cuda_full_version": "12.1.1", "cudnn_version": "8", "image_type": "runtime", "docker": "ghcr.io/pytorch/pytorch-nightly:2.4.0.dev20240410-cuda12.1-cudnn8-runtime", "platform": "linux/arm64,linux/amd64"}, {"cuda": "12.1", "cuda_full_version": "12.1.1", "cudnn_version": "8", "image_type": "devel", "docker": "ghcr.io/pytorch/pytorch-nightly:2.4.0.dev20240410-cuda12.1-cudnn8-devel", "platform": "linux/arm64,linux/amd64"}]}
(.venv) [[email protected] /data/users/atalman/test-infra/tools/scripts (generate_docker_matrix)]$ python generate_docker_release_matrix.py
{"include": [{"cuda": "11.8", "cuda_full_version": "11.8.0", "cudnn_version": "8", "image_type": "runtime", "docker": "ghcr.io/pytorch/pytorch-nightly:2.4.0.dev20240410-cuda11.8-cudnn8-runtime", "platform": "linux/arm64,linux/amd64"}, {"cuda": "11.8", "cuda_full_version": "11.8.0", "cudnn_version": "8", "image_type": "devel", "docker": "ghcr.io/pytorch/pytorch-nightly:2.4.0.dev20240410-cuda11.8-cudnn8-devel", "platform": "linux/arm64,linux/amd64"}, {"cuda": "12.1", "cuda_full_version": "12.1.1", "cudnn_version": "8", "image_type": "runtime", "docker": "ghcr.io/pytorch/pytorch-nightly:2.4.0.dev20240410-cuda12.1-cudnn8-runtime", "platform": "linux/arm64,linux/amd64"}, {"cuda": "12.1", "cuda_full_version": "12.1.1", "cudnn_version": "8", "image_type": "devel", "docker": "ghcr.io/pytorch/pytorch-nightly:2.4.0.dev20240410-cuda12.1-cudnn8-devel", "platform": "linux/arm64,linux/amd64"}]}
```
  • Loading branch information
@atalman
atalman authored Apr 10, 2024
1 parent 751efc1 commit 3259e39
Show file tree
Hide file tree
Showing 2 changed files with 13 additions and 0 deletions.
1 change: 1 addition & 0 deletions tools/scripts/generate_binary_build_matrix.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@
ROCM = "rocm"


CURRENT_NIGHTLY_VERSION = "2.4.0"
CURRENT_CANDIDATE_VERSION = "2.3.0"
CURRENT_STABLE_VERSION = "2.2.2"
mod.CURRENT_VERSION = CURRENT_STABLE_VERSION
Expand Down
12 changes: 12 additions & 0 deletions tools/scripts/generate_docker_release_matrix.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
import sys
import argparse
from typing import Dict, List
from datetime import datetime

import generate_binary_build_matrix

Expand All @@ -27,13 +28,24 @@ def generate_docker_matrix(channel: str) -> Dict[str, List[Dict[str, str]]]:
ret: List[Dict[str, str]] = []
for cuda in generate_binary_build_matrix.CUDA_ARCHES_DICT[channel]:
version = generate_binary_build_matrix.CUDA_CUDDN_VERSIONS[cuda]

prefix = "ghcr.io/pytorch/pytorch"
docker_image_version = ""
if channel == "release":
docker_image_version = f"{prefix}:{generate_binary_build_matrix.CURRENT_STABLE_VERSION}"
elif channel == "test":
docker_image_version = f"{prefix}-test:{generate_binary_build_matrix.CURRENT_CANDIDATE_VERSION}"
else:
docker_image_version = f"{prefix}-nightly:{generate_binary_build_matrix.CURRENT_NIGHTLY_VERSION}.dev{datetime.today().strftime('%Y%m%d')}"

for image in DOCKER_IMAGE_TYPES:
ret.append(
{
"cuda": cuda,
"cuda_full_version": version["cuda"],
"cudnn_version": version["cudnn"],
"image_type": image,
"docker": f"{docker_image_version}-cuda{cuda}-cudnn{version['cudnn']}-{image}",
"platform": "linux/arm64,linux/amd64",
}
)
Expand Down

0 comments on commit 3259e39

Please sign in to comment.