Revert "Revert "DO NOT REVERT - Updating workflows to gather AWS auth…

…entication via OIDC" (#4785)" This reverts commit 3a745d2.
pytorch · Dec 19, 2023 · 260140e · 260140e
1 parent f702f81
commit 260140e
Show file tree

Hide file tree

Showing 14 changed files with 142 additions and 108 deletions.
diff --git a/.github/workflows/backfill-workflow-job.yml b/.github/workflows/backfill-workflow-job.yml
@@ -12,6 +12,11 @@ permissions:
 defaults:
   run:
     working-directory: torchci
+
+permissions:
+  id-token: write
+  contents: read
+
 jobs:
   backfill-rockset:
     runs-on: ubuntu-20.04

diff --git a/.github/workflows/build_wheels_linux.yml b/.github/workflows/build_wheels_linux.yml
@@ -65,13 +65,10 @@ on:
         required: false
         type: boolean
         default: true
-    secrets:
-      AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID:
-        description: "AWS Access Key passed from caller workflow"
-        required: false
-      AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY:
-        description: "AWS Secret Access Ket passed from caller workflow"
-        required: false
+
+permissions:
+  id-token: write
+  contents: read
 
 jobs:
   build:
@@ -122,6 +119,11 @@ jobs:
           repository: "pytorch/builder"
           ref: "main"
           path: builder
+      - name: configure aws credentials
+        uses: aws-actions/[email protected]
+        with:
+          role-to-assume: arn:aws:iam::308535385114:role/gha_workflow_build_wheels_linux
+          aws-region: us-east-1
       - name: Set linux aarch64 CI
         if: ${{ inputs.architecture == 'aarch64' }}
         shell: bash -l {0}
@@ -234,9 +236,6 @@ jobs:
         if: ${{ (inputs.trigger-event == 'push' && startsWith(github.event.ref, 'refs/heads/nightly')) || (env.CHANNEL == 'test' && startsWith(github.event.ref, 'refs/tags/')) }}
         shell: bash -l {0}
         working-directory: ${{ inputs.repository }}
-        env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }}
         run: |
           set -euxo pipefail
           source "${BUILD_ENV_FILE}"

diff --git a/.github/workflows/build_wheels_macos.yml b/.github/workflows/build_wheels_macos.yml
@@ -59,13 +59,10 @@ on:
         description: "The key created when saving a cache and the key used to search for a cache."
         default: ""
         type: string
-    secrets:
-      AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID:
-        description: "AWS Access Key passed from caller workflow"
-        required: false
-      AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY:
-        description: "AWS Secret Access Ket passed from caller workflow"
-        required: false
+
+permissions:
+  id-token: write
+  contents: read
 
 jobs:
   build:
@@ -97,6 +94,11 @@ jobs:
           repository: ${{ inputs.test-infra-repository }}
           ref: ${{ inputs.test-infra-ref }}
           path: test-infra
+      - name: configure aws credentials
+        uses: aws-actions/[email protected]
+        with:
+          role-to-assume: arn:aws:iam::308535385114:role/gha_workflow_build_wheels_macos
+          aws-region: us-east-1
       - uses: ./test-infra/.github/actions/set-channel
       - name: Set PYTORCH_VERSION
         if: ${{ env.CHANNEL == 'test' }}
@@ -208,9 +210,6 @@ jobs:
         if: ${{ (inputs.trigger-event == 'push' && startsWith(github.event.ref, 'refs/heads/nightly')) || (env.CHANNEL == 'test' && startsWith(github.event.ref, 'refs/tags/')) }}
         shell: bash -l {0}
         working-directory: ${{ inputs.repository }}
-        env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }}
         run: |
           set -euxo pipefail
           # shellcheck disable=SC1090

diff --git a/.github/workflows/build_wheels_windows.yml b/.github/workflows/build_wheels_windows.yml
@@ -59,13 +59,10 @@ on:
         description: "The key created when saving a cache and the key used to search for a cache."
         default: ""
         type: string
-    secrets:
-      AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID:
-        description: "AWS Access Key passed from caller workflow"
-        required: false
-      AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY:
-        description: "AWS Secret Access Ket passed from caller workflow"
-        required: false
+
+permissions:
+  id-token: write
+  contents: read
 
 jobs:
   build:
@@ -194,13 +191,15 @@ jobs:
             echo "${{ inputs.repository }}/${SMOKE_TEST_SCRIPT} found"
             ${CONDA_RUN} python "${{ inputs.repository }}/${SMOKE_TEST_SCRIPT}"
           fi
+      - name: configure aws credentials
+        uses: aws-actions/[email protected]
+        with:
+          role-to-assume: arn:aws:iam::308535385114:role/gha_workflow_build_wheels_windows
+          aws-region: us-east-1
       - name: Upload package to pytorch.org
         if: ${{ (inputs.trigger-event == 'push' && startsWith(github.event.ref, 'refs/heads/nightly')) || (env.CHANNEL == 'test' && startsWith(github.event.ref, 'refs/tags/')) }}
         shell: bash -l {0}
         working-directory: ${{ inputs.repository }}
-        env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }}
         run: |
           source "${BUILD_ENV_FILE}"
           ${CONDA_RUN} pip install awscli

diff --git a/.github/workflows/clang-tidy-linux.yml b/.github/workflows/clang-tidy-linux.yml
@@ -14,6 +14,10 @@ on:
       - '!tools/clang-tidy-checks/README.md'
       - '.github/workflows/clang-tidy-linux.yml'
 
+permissions:
+  id-token: write
+  contents: read
+
 jobs:
   build:
     runs-on: linux.12xlarge
@@ -37,6 +41,13 @@ jobs:
           docker cp "$image_id":/clang-tidy-checks/build/bin/clang-tidy ./clang-tidy
           docker cp "$image_id":/clang-tidy-checks/build/bin/clang-format ./clang-format
           docker rm -v "$image_id"
+      - name: configure aws credentials
+        uses: aws-actions/[email protected]
+        id: aws_creds
+        with:
+          role-to-assume: arn:aws:iam::308535385114:role/gha_workflow_clang-tidy-linux
+          aws-region: us-east-1
+          output-credentials: true
       - uses: driazati/upload-artifact-s3@50adbe4ef0b6d9221df25c18c5fc528dfcb7c3f8
         name: Publish clang-tidy binary
         if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
@@ -46,19 +57,8 @@ jobs:
           s3-prefix: linux64/17.0.6
           s3-bucket: oss-clang-format
           path: clang-tidy
-          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-      - uses: driazati/upload-artifact-s3@50adbe4ef0b6d9221df25c18c5fc528dfcb7c3f8
-        name: Publish clang-format binary
-        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
-        with:
-          name: clang-format
-          if-no-files-found: error
-          s3-prefix: linux64/17.0.6
-          s3-bucket: oss-clang-format
-          path: clang-format
-          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws_access_key_id: ${{ steps.aws_creds.outputs.aws-access-key-id }}
+          aws_secret_access_key: ${{ steps.aws_creds.outputs.aws-secret-access-key }}
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}

diff --git a/.github/workflows/clang-tidy-macos.yml b/.github/workflows/clang-tidy-macos.yml
@@ -16,6 +16,10 @@ on:
       - '!tools/clang-tidy-checks/README.md'
       - '.github/workflows/clang-tidy-macos.yml'
 
+permissions:
+  id-token: write
+  contents: read
+
 jobs:
   build-Intel:
     runs-on: macos-12-xl
@@ -38,6 +42,13 @@ jobs:
           export PATH
 
           ./setup.sh
+      - name: configure aws credentials
+        id: aws_creds
+        uses: aws-actions/[email protected]
+        with:
+          role-to-assume: arn:aws:iam::308535385114:role/gha_workflow_clang-tidy-macos
+          aws-region: us-east-1
+          output-credentials: true
       - uses: driazati/upload-artifact-s3@50adbe4ef0b6d9221df25c18c5fc528dfcb7c3f8
         name: Publish clang-tidy binary
         if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
@@ -47,19 +58,8 @@ jobs:
           s3-prefix: macos-i386/17.0.6
           s3-bucket: oss-clang-format
           path: tools/clang-tidy-checks/llvm-project/build/bin/clang-tidy
-          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-      - uses: driazati/upload-artifact-s3@50adbe4ef0b6d9221df25c18c5fc528dfcb7c3f8
-        name: Publish clang-format binary
-        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
-        with:
-          name: clang-format
-          if-no-files-found: error
-          s3-prefix: macos-i386/17.0.6
-          s3-bucket: oss-clang-format
-          path: tools/clang-tidy-checks/llvm-project/build/bin/clang-format
-          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws_access_key_id: ${{ steps.aws_creds.outputs.aws-access-key-id }}
+          aws_secret_access_key: ${{ steps.aws_creds.outputs.aws-secret-access-key }}
   build-M1:
     runs-on: macos-m1-12
     steps:
@@ -78,6 +78,13 @@ jobs:
           export PATH
 
           ./setup.sh
+      - name: configure aws credentials
+        id: aws_creds
+        uses: aws-actions/[email protected]
+        with:
+          role-to-assume: arn:aws:iam::308535385114:role/gha_workflow_clang-tidy-macos
+          aws-region: us-east-1
+          output-credentials: true
       - uses: driazati/upload-artifact-s3@50adbe4ef0b6d9221df25c18c5fc528dfcb7c3f8
         name: Publish clang-tidy binary
         if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
@@ -87,19 +94,8 @@ jobs:
           s3-prefix: macos-arm/17.0.6
           s3-bucket: oss-clang-format
           path: tools/clang-tidy-checks/llvm-project/build/bin/clang-tidy
-          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-      - uses: driazati/upload-artifact-s3@50adbe4ef0b6d9221df25c18c5fc528dfcb7c3f8
-        name: Publish clang-format binary
-        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
-        with:
-          name: clang-format
-          if-no-files-found: error
-          s3-prefix: macos-arm/17.0.6
-          s3-bucket: oss-clang-format
-          path: tools/clang-tidy-checks/llvm-project/build/bin/clang-format
-          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws_access_key_id: ${{ steps.aws_creds.outputs.aws-access-key-id }}
+          aws_secret_access_key: ${{ steps.aws_creds.outputs.aws-secret-access-key }}
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}

diff --git a/.github/workflows/github-status-test-lambda.yml b/.github/workflows/github-status-test-lambda.yml
@@ -16,6 +16,10 @@ defaults:
   run:
     working-directory: aws/lambda/github-status-test/
 
+permissions:
+  id-token: write
+  contents: read
+
 jobs:
   deploy:
     runs-on: ubuntu-latest
@@ -36,13 +40,15 @@ jobs:
           command: |
             set -eux
             python3 -m pip install awscli
+      - name: configure aws credentials
+        uses: aws-actions/[email protected]
+        with:
+          role-to-assume: arn:aws:iam::308535385114:role/gha_workflow_github-status-test-lambda
+          aws-region: us-east-1
       - name: Prepare package
         run: |
           make prepare
       - name: Deploy the lambda. (This only makes the lambda ready. Manual verification steps are still required to get it into prod because of the lack of automated testing here. Checkout aws/lambda/github-status-test/github-status-test/README.md to see how to get the deployed lambda to prod)
         if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
-        env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
         run: |
           make deploy
diff --git a/.github/workflows/lambda_github_status_webhook_handler.yml b/.github/workflows/lambda_github_status_webhook_handler.yml
@@ -8,17 +8,28 @@ on:
       - '.github/workflows/lambda_github_status_webhook_handler.yml'
       - 'aws/lambda/github-status-webhook-handler/**'
 
+permissions:
+  id-token: write
+  contents: read
+
 jobs:
   deploy:
     runs-on: ubuntu-20.04
     steps:
       - name: Checkout
         uses: actions/checkout@v3
+      - name: configure aws credentials
+        id: aws_creds
+        uses: aws-actions/[email protected]
+        with:
+          role-to-assume: arn:aws:iam::308535385114:role/gha_workflow_lambda_github_status_webhook_handler
+          aws-region: us-east-1
+          output-credentials: true
       - name: Deploy
         uses: appleboy/lambda-action@1e05c1377056f21ebb2ce69b910bc16b943c2a66
         with:
-          aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws_access_key_id: ${{ steps.aws_creds.outputs.aws-access-key-id }}
+          aws_secret_access_key: ${{ steps.aws_creds.outputs.aws-secret-access-key }}
           aws_region: us-east-1
           function_name: github-status-webhook-handler
           source: aws/lambda/github-status-webhook-handler/lambda_function.py

diff --git a/.github/workflows/log-classifier-lambda.yml b/.github/workflows/log-classifier-lambda.yml
@@ -15,6 +15,10 @@ defaults:
   run:
     working-directory: aws/lambda/log-classifier/
 
+permissions:
+  id-token: write
+  contents: read
+
 jobs:
   test:
     runs-on: ubuntu-20.04
@@ -29,9 +33,11 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v2
+      - name: configure aws credentials
+        uses: aws-actions/[email protected]
+        with:
+          role-to-assume: arn:aws:iam::308535385114:role/gha_workflow_log-classifier-lambda
+          aws-region: us-east-1
       - run: pip3 install cargo-lambda
       - run: cargo lambda build --release
       - run: cargo lambda deploy
-        env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
diff --git a/.github/workflows/opensearch-gha-jobs-lambda.yml b/.github/workflows/opensearch-gha-jobs-lambda.yml
@@ -15,6 +15,10 @@ defaults:
   run:
     working-directory: aws/lambda/opensearch-gha-jobs/
 
+permissions:
+  id-token: write
+  contents: read
+
 jobs:
   test:
     runs-on: ubuntu-22.04
@@ -37,7 +41,9 @@ jobs:
         with:
           python-version: '3.11'
           cache: pip
+      - name: configure aws credentials
+        uses: aws-actions/[email protected]
+        with:
+          role-to-assume: arn:aws:iam::308535385114:role/gha_workflow_opensearch-gha-jobs-lambda
+          aws-region: us-east-1
       - run: make deploy
-        env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
diff --git a/.github/workflows/update-slow-tests.yml b/.github/workflows/update-slow-tests.yml
@@ -14,6 +14,11 @@ permissions:
 defaults:
   run:
     working-directory: torchci
+
+permissions:
+  id-token: write
+  contents: read
+
 jobs:
   update-slow-stats:
     runs-on: ubuntu-20.04