Specific PYTORCH_VERSION when building RC for test channel (#4804)

Cherry pick #4803 for 2.1.2
release. This is to fix the issue when building 2.1.2 RC for domains
pick up 2.2.0 from test channel instead of 2.1.2

The changes in `tools/tests/assets` are not really needed but I like a
green signal for `python3 -m unittest discover -vs tools/tests -p
'test_*.py'`. The differences are in the pinned Docker images.

.github/actions/set-channel/action.yml

@@ -6,11 +6,6 @@ description: Add CHANNEL to GITHUB_ENV
 runs:
   using: composite
   steps:
-    - name: Set default CHANNEL
-      shell: bash
-      run: |
-        set -euxo pipefail
-        echo "CHANNEL=nightly" >> "${GITHUB_ENV}"
     - name: Set CHANNEL for tagged pushes
       if: ${{ github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/') }}
       shell: bash
@@ -26,3 +21,12 @@ runs:
       run: |
         set -euxo pipefail
         echo "CHANNEL=test" >> "$GITHUB_ENV"
+    - name: Set default CHANNEL
+      # Set this to nightly only when the CHANNEL hasn't been set yet. In GHA, once
+      # an env is set, it's fixed unless we can figure out a way to overwrite it in
+      # $GITHUB_ENV
+      if: ${{ env.CHANNEL == '' }}
+      shell: bash
+      run: |
+        set -euxo pipefail
+        echo "CHANNEL=nightly" >> "${GITHUB_ENV}"

.github/workflows/build_conda_linux.yml

@@ -100,6 +100,12 @@ jobs:
         with:
           github-secret: ${{ github.token }}
       - uses: ./test-infra/.github/actions/set-channel
+      - name: Set PYTORCH_VERSION
+        if: ${{ env.CHANNEL == 'test' }}
+        run: |
+          # When building RC, set the version to be the current candidate version,
+          # otherwise, leave it alone so nightly will pick up the latest
+          echo "PYTORCH_VERSION=${{ matrix.stable_version }}" >> "${GITHUB_ENV}"
       - uses: ./test-infra/.github/actions/setup-binary-builds
         with:
           repository: ${{ inputs.repository }}

.github/workflows/build_conda_macos.yml

@@ -106,6 +106,12 @@ jobs:
       - name: Set env variable for architecture name
         run: echo "ARCH_NAME=$(uname -m)" >> "${GITHUB_ENV}"
       - uses: ./test-infra/.github/actions/set-channel
+      - name: Set PYTORCH_VERSION
+        if: ${{ env.CHANNEL == 'test' }}
+        run: |
+          # When building RC, set the version to be the current candidate version,
+          # otherwise, leave it alone so nightly will pick up the latest
+          echo "PYTORCH_VERSION=${{ matrix.stable_version }}" >> "${GITHUB_ENV}"
       - uses: ./test-infra/.github/actions/setup-binary-builds
         with:
           repository: ${{ inputs.repository }}

.github/workflows/build_conda_windows.yml

@@ -96,6 +96,12 @@ jobs:
           ref: ${{ inputs.test-infra-ref }}
           path: test-infra
       - uses: ./test-infra/.github/actions/set-channel
+      - name: Set PYTORCH_VERSION
+        if: ${{ env.CHANNEL == 'test' }}
+        run: |
+          # When building RC, set the version to be the current candidate version,
+          # otherwise, leave it alone so nightly will pick up the latest
+          echo "PYTORCH_VERSION=${{ matrix.stable_version }}" >> "${GITHUB_ENV}"
       - uses: ./test-infra/.github/actions/setup-ssh
         name: Setup SSH
         with:

.github/workflows/build_wheels_linux.yml

@@ -138,6 +138,12 @@ jobs:
           echo "/opt/conda/bin" >> $GITHUB_PATH
           set -e
       - uses: ./test-infra/.github/actions/set-channel
+      - name: Set PYTORCH_VERSION
+        if: ${{ env.CHANNEL == 'test' }}
+        run: |
+          # When building RC, set the version to be the current candidate version,
+          # otherwise, leave it alone so nightly will pick up the latest
+          echo "PYTORCH_VERSION=${{ matrix.stable_version }}" >> "${GITHUB_ENV}"
       - uses: ./test-infra/.github/actions/setup-binary-builds
         env:
           PLATFORM: ${{ inputs.architecture == 'aarch64'  && 'linux-aarch64' || ''}}

.github/workflows/build_wheels_macos.yml

@@ -98,6 +98,12 @@ jobs:
           ref: ${{ inputs.test-infra-ref }}
           path: test-infra
       - uses: ./test-infra/.github/actions/set-channel
+      - name: Set PYTORCH_VERSION
+        if: ${{ env.CHANNEL == 'test' }}
+        run: |
+          # When building RC, set the version to be the current candidate version,
+          # otherwise, leave it alone so nightly will pick up the latest
+          echo "PYTORCH_VERSION=${{ matrix.stable_version }}" >> "${GITHUB_ENV}"
       - uses: ./test-infra/.github/actions/setup-binary-builds
         with:
           repository: ${{ inputs.repository }}

.github/workflows/build_wheels_windows.yml

@@ -105,6 +105,12 @@ jobs:
         run: |
           echo "C:/Jenkins/Miniconda3/Scripts" >> $GITHUB_PATH
       - uses: ./test-infra/.github/actions/set-channel
+      - name: Set PYTORCH_VERSION
+        if: ${{ env.CHANNEL == 'test' }}
+        run: |
+          # When building RC, set the version to be the current candidate version,
+          # otherwise, leave it alone so nightly will pick up the latest
+          echo "PYTORCH_VERSION=${{ matrix.stable_version }}" >> "${GITHUB_ENV}"
       - uses: ./test-infra/.github/actions/setup-binary-builds
         with:
           repository: ${{ inputs.repository }}

.github/workflows/generate_binary_build_matrix.yml

@@ -13,7 +13,7 @@ on:
         type: string
       channel:
         description: "Channel to use (nightly, test, release, all)"
-        default: "nightly"
+        default: ""
         type: string
       test-infra-repository:
         description: "Test infra repository to use"
@@ -60,7 +60,7 @@ jobs:
         env:
           PACKAGE_TYPE: ${{ inputs.package-type }}
           OS: ${{ inputs.os }}
-          CHANNEL: ${{ inputs.channel }}
+          CHANNEL: ${{ inputs.channel != '' && inputs.channel || env.CHANNEL }}
           WITH_CUDA: ${{ inputs.with-cuda }}
           WITH_ROCM: ${{ inputs.with-rocm }}
           WITH_CPU: ${{ inputs.with-cpu }}

tools/scripts/generate_binary_build_matrix.py

@@ -45,8 +45,21 @@
 NIGHTLY = "nightly"
 TEST = "test"
 
-CURRENT_CANDIDATE_VERSION = "2.1.0"
-CURRENT_STABLE_VERSION = "2.0.1"
+# OS constants
+LINUX = "linux"
+LINUX_AARCH64 = "linux-aarch64"
+MACOS = "macos"
+MACOS_ARM64 = "macos-arm64"
+WINDOWS = "windows"
+
+# Accelerator architectures
+CPU = "cpu"
+CPU_AARCH64 = "cpu-aarch64"
+CUDA = "cuda"
+ROCM = "rocm"
+
+CURRENT_CANDIDATE_VERSION = "2.1.2"
+CURRENT_STABLE_VERSION = "2.1.1"
 mod.CURRENT_VERSION = CURRENT_STABLE_VERSION
 
 # By default use Nightly for CUDA arches

tools/tests/assets/build_matrix_linux_conda_cuda.json

@@ -1 +1 @@
-{"include": [{"python_version": "3.8", "gpu_arch_type": "cpu", "gpu_arch_version": "", "desired_cuda": "cpu", "container_image": "pytorch/conda-builder:cpu", "package_type": "conda", "build_name": "conda-py3_8-cpu", "validation_runner": "linux.2xlarge", "channel": "nightly", "stable_version": "2.0.1", "installation": "conda install pytorch torchvision torchaudio cpuonly -c pytorch-nightly"}, {"python_version": "3.8", "gpu_arch_type": "cuda", "gpu_arch_version": "11.8", "desired_cuda": "cu118", "container_image": "pytorch/conda-builder:cuda11.8", "package_type": "conda", "build_name": "conda-py3_8-cuda11_8", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "channel": "nightly", "stable_version": "2.0.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=11.8 -c pytorch-nightly -c nvidia"}, {"python_version": "3.8", "gpu_arch_type": "cuda", "gpu_arch_version": "12.1", "desired_cuda": "cu121", "container_image": "pytorch/conda-builder:cuda12.1", "package_type": "conda", "build_name": "conda-py3_8-cuda12_1", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "channel": "nightly", "stable_version": "2.0.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=12.1 -c pytorch-nightly -c nvidia"}, {"python_version": "3.9", "gpu_arch_type": "cpu", "gpu_arch_version": "", "desired_cuda": "cpu", "container_image": "pytorch/conda-builder:cpu", "package_type": "conda", "build_name": "conda-py3_9-cpu", "validation_runner": "linux.2xlarge", "channel": "nightly", "stable_version": "2.0.1", "installation": "conda install pytorch torchvision torchaudio cpuonly -c pytorch-nightly"}, {"python_version": "3.9", "gpu_arch_type": "cuda", "gpu_arch_version": "11.8", "desired_cuda": "cu118", "container_image": "pytorch/conda-builder:cuda11.8", "package_type": "conda", "build_name": "conda-py3_9-cuda11_8", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "channel": "nightly", "stable_version": "2.0.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=11.8 -c pytorch-nightly -c nvidia"}, {"python_version": "3.9", "gpu_arch_type": "cuda", "gpu_arch_version": "12.1", "desired_cuda": "cu121", "container_image": "pytorch/conda-builder:cuda12.1", "package_type": "conda", "build_name": "conda-py3_9-cuda12_1", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "channel": "nightly", "stable_version": "2.0.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=12.1 -c pytorch-nightly -c nvidia"}, {"python_version": "3.10", "gpu_arch_type": "cpu", "gpu_arch_version": "", "desired_cuda": "cpu", "container_image": "pytorch/conda-builder:cpu", "package_type": "conda", "build_name": "conda-py3_10-cpu", "validation_runner": "linux.2xlarge", "channel": "nightly", "stable_version": "2.0.1", "installation": "conda install pytorch torchvision torchaudio cpuonly -c pytorch-nightly"}, {"python_version": "3.10", "gpu_arch_type": "cuda", "gpu_arch_version": "11.8", "desired_cuda": "cu118", "container_image": "pytorch/conda-builder:cuda11.8", "package_type": "conda", "build_name": "conda-py3_10-cuda11_8", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "channel": "nightly", "stable_version": "2.0.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=11.8 -c pytorch-nightly -c nvidia"}, {"python_version": "3.10", "gpu_arch_type": "cuda", "gpu_arch_version": "12.1", "desired_cuda": "cu121", "container_image": "pytorch/conda-builder:cuda12.1", "package_type": "conda", "build_name": "conda-py3_10-cuda12_1", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "channel": "nightly", "stable_version": "2.0.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=12.1 -c pytorch-nightly -c nvidia"}, {"python_version": "3.11", "gpu_arch_type": "cpu", "gpu_arch_version": "", "desired_cuda": "cpu", "container_image": "pytorch/conda-builder:cpu", "package_type": "conda", "build_name": "conda-py3_11-cpu", "validation_runner": "linux.2xlarge", "channel": "nightly", "stable_version": "2.0.1", "installation": "conda install pytorch torchvision torchaudio cpuonly -c pytorch-nightly"}, {"python_version": "3.11", "gpu_arch_type": "cuda", "gpu_arch_version": "11.8", "desired_cuda": "cu118", "container_image": "pytorch/conda-builder:cuda11.8", "package_type": "conda", "build_name": "conda-py3_11-cuda11_8", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "channel": "nightly", "stable_version": "2.0.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=11.8 -c pytorch-nightly -c nvidia"}, {"python_version": "3.11", "gpu_arch_type": "cuda", "gpu_arch_version": "12.1", "desired_cuda": "cu121", "container_image": "pytorch/conda-builder:cuda12.1", "package_type": "conda", "build_name": "conda-py3_11-cuda12_1", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "channel": "nightly", "stable_version": "2.0.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=12.1 -c pytorch-nightly -c nvidia"}]}
+{"include": [{"python_version": "3.8", "gpu_arch_type": "cpu", "gpu_arch_version": "", "desired_cuda": "cpu", "container_image": "pytorch/conda-builder:cpu-941be28cb5c686dc41b7ea8681701e64192c3002", "package_type": "conda", "build_name": "conda-py3_8-cpu", "validation_runner": "linux.2xlarge", "channel": "nightly", "stable_version": "2.1.1", "installation": "conda install pytorch torchvision torchaudio cpuonly -c pytorch-nightly"}, {"python_version": "3.8", "gpu_arch_type": "cuda", "gpu_arch_version": "11.8", "desired_cuda": "cu118", "container_image": "pytorch/conda-builder:cuda11.8-941be28cb5c686dc41b7ea8681701e64192c3002", "package_type": "conda", "build_name": "conda-py3_8-cuda11_8", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "channel": "nightly", "stable_version": "2.1.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=11.8 -c pytorch-nightly -c nvidia"}, {"python_version": "3.8", "gpu_arch_type": "cuda", "gpu_arch_version": "12.1", "desired_cuda": "cu121", "container_image": "pytorch/conda-builder:cuda12.1-941be28cb5c686dc41b7ea8681701e64192c3002", "package_type": "conda", "build_name": "conda-py3_8-cuda12_1", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "channel": "nightly", "stable_version": "2.1.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=12.1 -c pytorch-nightly -c nvidia"}, {"python_version": "3.9", "gpu_arch_type": "cpu", "gpu_arch_version": "", "desired_cuda": "cpu", "container_image": "pytorch/conda-builder:cpu-941be28cb5c686dc41b7ea8681701e64192c3002", "package_type": "conda", "build_name": "conda-py3_9-cpu", "validation_runner": "linux.2xlarge", "channel": "nightly", "stable_version": "2.1.1", "installation": "conda install pytorch torchvision torchaudio cpuonly -c pytorch-nightly"}, {"python_version": "3.9", "gpu_arch_type": "cuda", "gpu_arch_version": "11.8", "desired_cuda": "cu118", "container_image": "pytorch/conda-builder:cuda11.8-941be28cb5c686dc41b7ea8681701e64192c3002", "package_type": "conda", "build_name": "conda-py3_9-cuda11_8", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "channel": "nightly", "stable_version": "2.1.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=11.8 -c pytorch-nightly -c nvidia"}, {"python_version": "3.9", "gpu_arch_type": "cuda", "gpu_arch_version": "12.1", "desired_cuda": "cu121", "container_image": "pytorch/conda-builder:cuda12.1-941be28cb5c686dc41b7ea8681701e64192c3002", "package_type": "conda", "build_name": "conda-py3_9-cuda12_1", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "channel": "nightly", "stable_version": "2.1.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=12.1 -c pytorch-nightly -c nvidia"}, {"python_version": "3.10", "gpu_arch_type": "cpu", "gpu_arch_version": "", "desired_cuda": "cpu", "container_image": "pytorch/conda-builder:cpu-941be28cb5c686dc41b7ea8681701e64192c3002", "package_type": "conda", "build_name": "conda-py3_10-cpu", "validation_runner": "linux.2xlarge", "channel": "nightly", "stable_version": "2.1.1", "installation": "conda install pytorch torchvision torchaudio cpuonly -c pytorch-nightly"}, {"python_version": "3.10", "gpu_arch_type": "cuda", "gpu_arch_version": "11.8", "desired_cuda": "cu118", "container_image": "pytorch/conda-builder:cuda11.8-941be28cb5c686dc41b7ea8681701e64192c3002", "package_type": "conda", "build_name": "conda-py3_10-cuda11_8", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "channel": "nightly", "stable_version": "2.1.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=11.8 -c pytorch-nightly -c nvidia"}, {"python_version": "3.10", "gpu_arch_type": "cuda", "gpu_arch_version": "12.1", "desired_cuda": "cu121", "container_image": "pytorch/conda-builder:cuda12.1-941be28cb5c686dc41b7ea8681701e64192c3002", "package_type": "conda", "build_name": "conda-py3_10-cuda12_1", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "channel": "nightly", "stable_version": "2.1.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=12.1 -c pytorch-nightly -c nvidia"}, {"python_version": "3.11", "gpu_arch_type": "cpu", "gpu_arch_version": "", "desired_cuda": "cpu", "container_image": "pytorch/conda-builder:cpu-941be28cb5c686dc41b7ea8681701e64192c3002", "package_type": "conda", "build_name": "conda-py3_11-cpu", "validation_runner": "linux.2xlarge", "channel": "nightly", "stable_version": "2.1.1", "installation": "conda install pytorch torchvision torchaudio cpuonly -c pytorch-nightly"}, {"python_version": "3.11", "gpu_arch_type": "cuda", "gpu_arch_version": "11.8", "desired_cuda": "cu118", "container_image": "pytorch/conda-builder:cuda11.8-941be28cb5c686dc41b7ea8681701e64192c3002", "package_type": "conda", "build_name": "conda-py3_11-cuda11_8", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "channel": "nightly", "stable_version": "2.1.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=11.8 -c pytorch-nightly -c nvidia"}, {"python_version": "3.11", "gpu_arch_type": "cuda", "gpu_arch_version": "12.1", "desired_cuda": "cu121", "container_image": "pytorch/conda-builder:cuda12.1-941be28cb5c686dc41b7ea8681701e64192c3002", "package_type": "conda", "build_name": "conda-py3_11-cuda12_1", "validation_runner": "linux.g5.4xlarge.nvidia.gpu", "channel": "nightly", "stable_version": "2.1.1", "installation": "conda install pytorch torchvision torchaudio pytorch-cuda=12.1 -c pytorch-nightly -c nvidia"}]}