Cherry pick a bunch of OIDC changes for 2.2.1 release (#4945)

The list includes: * #4870 * #4877 * #4882 * #4886 * #4891 * #4893 * #4894 * #4901 --------- Co-authored-by: Andrey Talman <[email protected]>
pytorch · Feb 12, 2024 · c76d973 · c76d973
1 parent 5d032b6
commit c76d973
Show file tree

Hide file tree

Showing 23 changed files with 479 additions and 221 deletions.
diff --git a/.github/actions/setup-binary-builds/action.yml b/.github/actions/setup-binary-builds/action.yml
@@ -7,9 +7,9 @@ inputs:
     description: If set to any value, don't use sudo to clean the workspace
     required: false
     type: string
-    default: ""
+    default: ''
   ref:
-    description: Works as stated in actions/checkout, but the default value is recursive
+    description: Works as stated in actions/checkout
     required: false
     type: string
     default: nightly
@@ -19,15 +19,27 @@ inputs:
     type: string
     default: recursive
   setup-miniconda:
-    description: Works as stated in actions/checkout, but the default value is recursive
+    description: Set to true if setup-miniconda is needed
     required: false
     type: boolean
     default: false
   python-version:
-    description: Works as stated in actions/checkout, but the default value is recursive
+    description: The target Python version
+    required: true
+    type: string
+  cuda-version:
+    description: The target CUDA version
+    required: true
+    type: string
+  arch:
+    description: The target ARCH
+    required: true
+    type: string
+  upload-to-base-bucket:
+    description: One of the parameter used by pkg-helpers
     required: false
     type: boolean
-    default: false
+    default: no
 
 runs:
   using: composite
@@ -62,11 +74,13 @@ runs:
         shell: bash
         env:
           PYTHON_VERSION: ${{ inputs.python-version }}
+          CU_VERSION: ${{ inputs.cuda-version }}
+          ARCH: ${{ inputs.arch }}
         run: |
           set -euxo pipefail
           # Set artifact name here since github actions doesn't have string manipulation tools
           # and "/" is not allowed in artifact names
-          echo "ARTIFACT_NAME=${REPOSITORY/\//_}_${REF}_${PYTHON_VERSION}" >> "${GITHUB_ENV}"
+          echo "ARTIFACT_NAME=${REPOSITORY/\//_}_${REF}_${PYTHON_VERSION}_${CU_VERSION}_${ARCH}" >> "${GITHUB_ENV}"
       - name: Setup miniconda (for pytorch_pkg_helpers)
         if: ${{ inputs.setup-miniconda == 'true' }}
         uses: conda-incubator/[email protected]

diff --git a/.github/actions/setup-binary-upload/action.yml b/.github/actions/setup-binary-upload/action.yml
@@ -0,0 +1,81 @@
+name: Set up binary upload jobs
+
+description: Setup a GitHub ephemeral runner to upload binary wheel and conda artifacts
+
+inputs:
+  repository:
+    description: The repository name, i.e. pytorch/vision
+    required: true
+    type: string
+  ref:
+    description: Part of the artifact name
+    required: false
+    type: string
+    default: ''
+  python-version:
+    description: Part of the artifact name
+    required: true
+    type: string
+  cuda-version:
+    description: Part of the artifact name
+    required: true
+    type: string
+  arch:
+    description: Part of the artifact name
+    required: true
+    type: string
+  upload-to-base-bucket:
+    description: One of the parameter used by pkg-helpers
+    required: false
+    type: boolean
+    default: no
+
+runs:
+  using: composite
+  steps:
+    - uses: actions/setup-python@v4
+      with:
+        python-version: '3.11'
+        cache: pip
+
+    - name: Set the artifact name
+      shell: bash
+      env:
+        REPOSITORY: ${{ inputs.repository }}
+        REF: ${{ inputs.ref }}
+        PYTHON_VERSION: ${{ inputs.python-version }}
+        CU_VERSION: ${{ inputs.cuda-version }}
+        ARCH: ${{ inputs.arch }}
+      run: |
+        set -ex
+
+        # Set artifact name here since github actions doesn't have string manipulation tools
+        # and "/" is not allowed in artifact names
+        echo "ARTIFACT_NAME=${REPOSITORY/\//_}_${REF}_${PYTHON_VERSION}_${CU_VERSION}_${ARCH}" >> "${GITHUB_ENV}"
+
+    # Need to checkout the target repository to run pkg-helpers
+    - uses: actions/checkout@v3
+      with:
+        repository: ${{ inputs.repository }}
+        ref: ${{ inputs.ref }}
+        path: ${{ inputs.repository }}
+
+    - name: Generate env variables from pytorch_pkg_helpers
+      working-directory: ${{ inputs.repository }}
+      shell: bash
+      env:
+        REPOSITORY: ${{ inputs.repository }}
+        REF: ${{ inputs.ref }}
+        PYTHON_VERSION: ${{ inputs.python-version }}
+        CU_VERSION: ${{ inputs.cuda-version }}
+        ARCH: ${{ inputs.arch }}
+      run: |
+        set -ex
+
+        python -m pip install ${GITHUB_WORKSPACE}/test-infra/tools/pkg-helpers
+
+        BUILD_ENV_FILE="${RUNNER_TEMP}/build_env_${GITHUB_RUN_ID}"
+        python -m pytorch_pkg_helpers > "${BUILD_ENV_FILE}"
+
+        cat "${BUILD_ENV_FILE}"
+        echo "BUILD_ENV_FILE=${BUILD_ENV_FILE}" >> "${GITHUB_ENV}"
diff --git a/.github/workflows/_binary_conda_upload.yml b/.github/workflows/_binary_conda_upload.yml
@@ -0,0 +1,112 @@
+name: upload conda
+
+on:
+  workflow_call:
+    inputs:
+      repository:
+        description: 'Repository to checkout, defaults to ""'
+        default: ''
+        type: string
+      ref:
+        description: 'Reference to checkout, defaults to "nightly"'
+        default: 'nightly'
+        type: string
+      test-infra-repository:
+        description: 'Test infra repository to use'
+        default: "pytorch/test-infra"
+        type: string
+      test-infra-ref:
+        description: 'Test infra reference to use'
+        default: ""
+        type: string
+      build-matrix:
+        description: 'Build matrix to utilize'
+        default: ''
+        type: string
+      trigger-event:
+        description: 'Trigger Event in caller that determines whether or not to upload'
+        type: string
+        default: ''
+    secrets:
+      CONDA_PYTORCHBOT_TOKEN:
+        description: 'Access Token needed to upload binaries to anaconda nightly channel'
+        required: false
+      CONDA_PYTORCHBOT_TOKEN_TEST:
+        description: 'Access Token needed to upload binaries to anaconda test channel'
+        required: false
+
+jobs:
+  upload:
+    runs-on: ubuntu-22.04
+    environment: ${{(inputs.trigger-event == 'push' && (startsWith(github.event.ref, 'refs/heads/nightly') || startsWith(github.event.ref, 'refs/tags/v'))) && 'pytorchbot-env' || ''}}
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(inputs.build-matrix) }}
+    timeout-minutes: 30
+    name: ${{ matrix.build_name }}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          repository: ${{ inputs.test-infra-repository }}
+          ref: ${{ inputs.test-infra-ref }}
+          path: test-infra
+
+      - uses: ./test-infra/.github/actions/set-channel
+
+      # For pytorch_pkg_helpers which we need to run to generate the artifact name and target S3 buckets
+      - uses: ./test-infra/.github/actions/setup-binary-upload
+        with:
+          repository: ${{ inputs.repository }}
+          ref: ${{ inputs.ref }}
+          python-version: ${{ matrix.python_version }}
+          cuda-version: ${{ matrix.desired_cuda }}
+          upload-to-base-bucket: ${{ matrix.upload_to_base_bucket }}
+
+      - name: Download the artifact
+        uses: actions/download-artifact@v3
+        with:
+          name: ${{ env.ARTIFACT_NAME }}
+          path: ${{ inputs.repository }}/distr
+
+      - name: Nightly or release RC
+        if: ${{ (inputs.trigger-event == 'push' && startsWith(github.event.ref, 'refs/heads/nightly')) || (env.CHANNEL == 'test' && startsWith(github.event.ref, 'refs/tags/')) }}
+        shell: bash
+        env:
+          CONDA_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_PYTORCHBOT_TOKEN }}
+          CONDA_PYTORCHBOT_TOKEN_TEST: ${{ secrets.CONDA_PYTORCHBOT_TOKEN_TEST }}
+        run: |
+          set -ex
+          echo "NIGHTLY_OR_TEST=1" >> "${GITHUB_ENV}"
+
+          if [[ "${CHANNEL}" = "nightly" ]]; then
+            echo "CONDA_TOKEN=${CONDA_PYTORCHBOT_TOKEN}" >> "${GITHUB_ENV}"
+          else
+            echo "CONDA_TOKEN=${CONDA_PYTORCHBOT_TOKEN_TEST}" >> "${GITHUB_ENV}"
+          fi
+
+      - name: Upload package to conda
+        working-directory: ${{ inputs.repository }}
+        run: |
+          set -ex
+
+          # shellcheck disable=SC1090
+          source "${BUILD_ENV_FILE}"
+          conda install --yes --quiet anaconda-client
+          conda run --no-capture-output anaconda --version
+
+          if [[ "${NIGHTLY_OR_TEST:-0}" == "1" ]]; then
+            for pkg in distr/**/*.tar.bz2; do
+              conda run --no-capture-output anaconda \
+                -t "${CONDA_TOKEN}" \
+                upload "${pkg}" \
+                -u "pytorch-${CHANNEL}" \
+                --label main \
+                --no-progress \
+                --force
+            done
+          else
+            echo "Testing the upload of the following files to pytorch-${CHANNEL} conda channel:"
+            for pkg in distr/**/*.tar.bz2; do
+              ls -lah "${pkg}"
+            done
+          fi
diff --git a/.github/workflows/_binary_upload.yml b/.github/workflows/_binary_upload.yml
@@ -0,0 +1,109 @@
+name: upload
+
+on:
+  workflow_call:
+    inputs:
+      repository:
+        description: 'Repository to checkout, defaults to ""'
+        default: ''
+        type: string
+      ref:
+        description: 'Reference to checkout, defaults to "nightly"'
+        default: 'nightly'
+        type: string
+      test-infra-repository:
+        description: "Test infra repository to use"
+        default: "pytorch/test-infra"
+        type: string
+      test-infra-ref:
+        description: "Test infra reference to use"
+        default: ""
+        type: string
+      build-matrix:
+        description: "Build matrix to utilize"
+        default: ''
+        type: string
+      architecture:
+        description: Architecture to build for x86_64 for default Linux, or aarch64 for Linux aarch64 builds
+        required: false
+        type: string
+        default: ''
+      trigger-event:
+        description: "Trigger Event in caller that determines whether or not to upload"
+        type: string
+        default: ''
+
+jobs:
+  upload:
+    runs-on: ubuntu-22.04
+    environment: ${{(inputs.trigger-event == 'push' && (startsWith(github.event.ref, 'refs/heads/nightly') || startsWith(github.event.ref, 'refs/tags/v'))) && 'pytorchbot-env' || ''}}
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(inputs.build-matrix) }}
+    timeout-minutes: 30
+    name: ${{ matrix.build_name }}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          repository: ${{ inputs.test-infra-repository }}
+          ref: ${{ inputs.test-infra-ref }}
+          path: test-infra
+
+      - uses: ./test-infra/.github/actions/set-channel
+
+      # For pytorch_pkg_helpers which we need to run to generate the artifact name and target S3 buckets
+      - uses: ./test-infra/.github/actions/setup-binary-upload
+        with:
+          repository: ${{ inputs.repository }}
+          ref: ${{ inputs.ref }}
+          python-version: ${{ matrix.python_version }}
+          cuda-version: ${{ matrix.desired_cuda }}
+          arch: ${{ inputs.architecture }}
+          upload-to-base-bucket: ${{ matrix.upload_to_base_bucket }}
+
+      - name: Download the artifact
+        uses: actions/download-artifact@v3
+        with:
+          name: ${{ env.ARTIFACT_NAME }}
+          path: ${{ inputs.repository }}/dist/
+
+      - name: Configure aws credentials (pytorch account)
+        if: ${{ inputs.trigger-event == 'push' && startsWith(github.event.ref, 'refs/heads/nightly') }}
+        uses: aws-actions/configure-aws-credentials@v3
+        with:
+          role-to-assume: arn:aws:iam::749337293305:role/gha_workflow_nightly_build_wheels
+          aws-region: us-east-1
+
+      - name: Configure aws credentials (pytorch account)
+        if: ${{ env.CHANNEL == 'test' && startsWith(github.event.ref, 'refs/tags/v') }}
+        uses: aws-actions/configure-aws-credentials@v3
+        with:
+          role-to-assume: arn:aws:iam::749337293305:role/gha_workflow_test_build_wheels
+          aws-region: us-east-1
+
+      - name: Nightly or release RC
+        if: ${{ (inputs.trigger-event == 'push' && startsWith(github.event.ref, 'refs/heads/nightly')) || (env.CHANNEL == 'test' && startsWith(github.event.ref, 'refs/tags/')) }}
+        shell: bash
+        run: |
+          set -ex
+          echo "NIGHTLY_OR_TEST=1" >> "${GITHUB_ENV}"
+
+      - name: Upload package to pytorch.org
+        shell: bash
+        working-directory: ${{ inputs.repository }}
+        run: |
+          set -ex
+
+          # shellcheck disable=SC1090
+          source "${BUILD_ENV_FILE}"
+
+          pip install awscli==1.32.18
+
+          AWS_CMD="aws s3 cp --dryrun"
+          if [[ "${NIGHTLY_OR_TEST:-0}" == "1" ]]; then
+            AWS_CMD="aws s3 cp"
+          fi
+
+          for pkg in dist/*; do
+            ${AWS_CMD} "$pkg" "${PYTORCH_S3_BUCKET_PATH}" --acl public-read
+          done