Reland "DO NOT REVERT - Updating workflows to gather AWS authentication via OIDC"

.github/workflows/backfill-workflow-job.yml

@@ -8,16 +8,24 @@ on:
 defaults:
   run:
     working-directory: torchci
+
+permissions:
+  id-token: write
+  contents: read
+
 jobs:
   backfill-rockset:
     runs-on: ubuntu-20.04
     steps:
+      - name: configure aws credentials
+        uses: aws-actions/[email protected]
+        with:
+          role-to-assume: arn:aws:iam::308535385114:role/gha_workflow_backfill-workflow-job
+          aws-region: us-east-1
       - uses: actions/checkout@v3
       - run: yarn install --frozen-lockfile
       - run: yarn node scripts/backfillJobs.mjs
         env:
           ROCKSET_API_KEY: ${{ secrets.ROCKSET_API_KEY }}
-          OUR_AWS_ACCESS_KEY_ID: ${{ secrets.OUR_AWS_ACCESS_KEY_ID }}
-          OUR_AWS_SECRET_ACCESS_KEY: ${{ secrets.OUR_AWS_SECRET_ACCESS_KEY }}
           APP_ID: ${{ secrets.APP_ID }}
           PRIVATE_KEY: ${{ secrets.PRIVATE_KEY }}

.github/workflows/build_wheels_linux.yml

@@ -65,13 +65,10 @@ on:
         required: false
         type: boolean
         default: true
-    secrets:
-      AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID:
-        description: "AWS Access Key passed from caller workflow"
-        required: false
-      AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY:
-        description: "AWS Secret Access Ket passed from caller workflow"
-        required: false
+
+permissions:
+  id-token: write
+  contents: read
 
 jobs:
   build:
@@ -122,6 +119,11 @@ jobs:
           repository: "pytorch/builder"
           ref: "main"
           path: builder
+      - name: configure aws credentials
+        uses: aws-actions/[email protected]
+        with:
+          role-to-assume: arn:aws:iam::308535385114:role/gha_workflow_build_wheels_linux
+          aws-region: us-east-1
       - name: Set linux aarch64 CI
         if: ${{ inputs.architecture == 'aarch64' }}
         shell: bash -l {0}
@@ -228,9 +230,6 @@ jobs:
         if: ${{ (inputs.trigger-event == 'push' && startsWith(github.event.ref, 'refs/heads/nightly')) || (env.CHANNEL == 'test' && startsWith(github.event.ref, 'refs/tags/')) }}
         shell: bash -l {0}
         working-directory: ${{ inputs.repository }}
-        env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }}
         run: |
           set -euxo pipefail
           source "${BUILD_ENV_FILE}"

.github/workflows/build_wheels_macos.yml

@@ -59,13 +59,10 @@ on:
         description: "The key created when saving a cache and the key used to search for a cache."
         default: ""
         type: string
-    secrets:
-      AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID:
-        description: "AWS Access Key passed from caller workflow"
-        required: false
-      AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY:
-        description: "AWS Secret Access Ket passed from caller workflow"
-        required: false
+
+permissions:
+  id-token: write
+  contents: read
 
 jobs:
   build:
@@ -97,6 +94,11 @@ jobs:
           repository: ${{ inputs.test-infra-repository }}
           ref: ${{ inputs.test-infra-ref }}
           path: test-infra
+      - name: configure aws credentials
+        uses: aws-actions/[email protected]
+        with:
+          role-to-assume: arn:aws:iam::308535385114:role/gha_workflow_build_wheels_macos
+          aws-region: us-east-1
       - uses: ./test-infra/.github/actions/set-channel
       - uses: ./test-infra/.github/actions/setup-binary-builds
         with:
@@ -202,9 +204,6 @@ jobs:
         if: ${{ (inputs.trigger-event == 'push' && startsWith(github.event.ref, 'refs/heads/nightly')) || (env.CHANNEL == 'test' && startsWith(github.event.ref, 'refs/tags/')) }}
         shell: bash -l {0}
         working-directory: ${{ inputs.repository }}
-        env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }}
         run: |
           set -euxo pipefail
           # shellcheck disable=SC1090

.github/workflows/build_wheels_windows.yml

@@ -59,13 +59,10 @@ on:
         description: "The key created when saving a cache and the key used to search for a cache."
         default: ""
         type: string
-    secrets:
-      AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID:
-        description: "AWS Access Key passed from caller workflow"
-        required: false
-      AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY:
-        description: "AWS Secret Access Ket passed from caller workflow"
-        required: false
+
+permissions:
+  id-token: write
+  contents: read
 
 jobs:
   build:
@@ -188,13 +185,15 @@ jobs:
             echo "${{ inputs.repository }}/${SMOKE_TEST_SCRIPT} found"
             ${CONDA_RUN} python "${{ inputs.repository }}/${SMOKE_TEST_SCRIPT}"
           fi
+      - name: configure aws credentials
+        uses: aws-actions/[email protected]
+        with:
+          role-to-assume: arn:aws:iam::308535385114:role/gha_workflow_build_wheels_windows
+          aws-region: us-east-1
       - name: Upload package to pytorch.org
         if: ${{ (inputs.trigger-event == 'push' && startsWith(github.event.ref, 'refs/heads/nightly')) || (env.CHANNEL == 'test' && startsWith(github.event.ref, 'refs/tags/')) }}
         shell: bash -l {0}
         working-directory: ${{ inputs.repository }}
-        env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }}
         run: |
           source "${BUILD_ENV_FILE}"
           ${CONDA_RUN} pip install awscli

.github/workflows/clang-tidy-linux.yml

@@ -14,6 +14,10 @@ on:
       - '!tools/clang-tidy-checks/README.md'
       - '.github/workflows/clang-tidy-linux.yml'
 
+permissions:
+  id-token: write
+  contents: read
+
 jobs:
   build:
     runs-on: linux.12xlarge
@@ -36,6 +40,13 @@ jobs:
           image_id=$(docker create ghcr.io/pytorch/cilint-clang-tidy:"$GITHUB_SHA")
           docker cp "$image_id":/clang-tidy-checks/build/bin/clang-tidy ./clang-tidy
           docker rm -v "$image_id"
+      - name: configure aws credentials
+        uses: aws-actions/[email protected]
+        id: aws_creds
+        with:
+          role-to-assume: arn:aws:iam::308535385114:role/gha_workflow_clang-tidy-linux
+          aws-region: us-east-1
+          output-credentials: true
       - uses: driazati/upload-artifact-s3@50adbe4ef0b6d9221df25c18c5fc528dfcb7c3f8
         name: Publish binary
         if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
@@ -45,8 +56,8 @@ jobs:
           s3-prefix: linux64/17.0.6
           s3-bucket: oss-clang-format
           path: clang-tidy
-          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws_access_key_id: ${{ steps.aws_creds.outputs.aws-access-key-id }}
+          aws_secret_access_key: ${{ steps.aws_creds.outputs.aws-secret-access-key }}
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}

.github/workflows/clang-tidy-macos.yml

@@ -16,6 +16,10 @@ on:
       - '!tools/clang-tidy-checks/README.md'
       - '.github/workflows/clang-tidy-macos.yml'
 
+permissions:
+  id-token: write
+  contents: read
+
 jobs:
   build-Intel:
     runs-on: macos-12-xl
@@ -38,6 +42,13 @@ jobs:
           export PATH
 
           ./setup.sh
+      - name: configure aws credentials
+        id: aws_creds
+        uses: aws-actions/[email protected]
+        with:
+          role-to-assume: arn:aws:iam::308535385114:role/gha_workflow_clang-tidy-macos
+          aws-region: us-east-1
+          output-credentials: true
       - uses: driazati/upload-artifact-s3@50adbe4ef0b6d9221df25c18c5fc528dfcb7c3f8
         name: Publish binary
         if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
@@ -47,8 +58,8 @@ jobs:
           s3-prefix: macos-i386/17.0.6
           s3-bucket: oss-clang-format
           path: tools/clang-tidy-checks/llvm-project/build/bin/clang-tidy
-          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws_access_key_id: ${{ steps.aws_creds.outputs.aws-access-key-id }}
+          aws_secret_access_key: ${{ steps.aws_creds.outputs.aws-secret-access-key }}
   build-M1:
     runs-on: macos-m1-12
     steps:
@@ -67,6 +78,13 @@ jobs:
           export PATH
 
           ./setup.sh
+      - name: configure aws credentials
+        id: aws_creds
+        uses: aws-actions/[email protected]
+        with:
+          role-to-assume: arn:aws:iam::308535385114:role/gha_workflow_clang-tidy-macos
+          aws-region: us-east-1
+          output-credentials: true
       - uses: driazati/upload-artifact-s3@50adbe4ef0b6d9221df25c18c5fc528dfcb7c3f8
         name: Publish binary
         if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
@@ -76,8 +94,8 @@ jobs:
           s3-prefix: macos-arm/17.0.6
           s3-bucket: oss-clang-format
           path: tools/clang-tidy-checks/llvm-project/build/bin/clang-tidy
-          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws_access_key_id: ${{ steps.aws_creds.outputs.aws-access-key-id }}
+          aws_secret_access_key: ${{ steps.aws_creds.outputs.aws-secret-access-key }}
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}

.github/workflows/github-status-test-lambda.yml

@@ -16,6 +16,10 @@ defaults:
   run:
     working-directory: aws/lambda/github-status-test/
 
+permissions:
+  id-token: write
+  contents: read
+
 jobs:
   deploy:
     runs-on: ubuntu-latest
@@ -36,13 +40,15 @@ jobs:
           command: |
             set -eux
             python3 -m pip install awscli
+      - name: configure aws credentials
+        uses: aws-actions/[email protected]
+        with:
+          role-to-assume: arn:aws:iam::308535385114:role/gha_workflow_github-status-test-lambda
+          aws-region: us-east-1
       - name: Prepare package
         run: |
           make prepare
       - name: Deploy the lambda. (This only makes the lambda ready. Manual verification steps are still required to get it into prod because of the lack of automated testing here. Checkout aws/lambda/github-status-test/github-status-test/README.md to see how to get the deployed lambda to prod)
         if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
-        env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
         run: |
           make deploy

.github/workflows/lambda_github_status_webhook_handler.yml

@@ -8,17 +8,28 @@ on:
       - '.github/workflows/lambda_github_status_webhook_handler.yml'
       - 'aws/lambda/github-status-webhook-handler/**'
 
+permissions:
+  id-token: write
+  contents: read
+
 jobs:
   deploy:
     runs-on: ubuntu-20.04
     steps:
       - name: Checkout
         uses: actions/checkout@v3
+      - name: configure aws credentials
+        id: aws_creds
+        uses: aws-actions/[email protected]
+        with:
+          role-to-assume: arn:aws:iam::308535385114:role/gha_workflow_lambda_github_status_webhook_handler
+          aws-region: us-east-1
+          output-credentials: true
       - name: Deploy
         uses: appleboy/lambda-action@1e05c1377056f21ebb2ce69b910bc16b943c2a66
         with:
-          aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws_access_key_id: ${{ steps.aws_creds.outputs.aws-access-key-id }}
+          aws_secret_access_key: ${{ steps.aws_creds.outputs.aws-secret-access-key }}
           aws_region: us-east-1
           function_name: github-status-webhook-handler
           source: aws/lambda/github-status-webhook-handler/lambda_function.py

.github/workflows/log-classifier-lambda.yml

@@ -15,6 +15,10 @@ defaults:
   run:
     working-directory: aws/lambda/log-classifier/
 
+permissions:
+  id-token: write
+  contents: read
+
 jobs:
   test:
     runs-on: ubuntu-20.04
@@ -29,9 +33,11 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v2
+      - name: configure aws credentials
+        uses: aws-actions/[email protected]
+        with:
+          role-to-assume: arn:aws:iam::308535385114:role/gha_workflow_log-classifier-lambda
+          aws-region: us-east-1
       - run: pip3 install cargo-lambda
       - run: cargo lambda build --release
       - run: cargo lambda deploy
-        env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}

.github/workflows/opensearch-gha-jobs-lambda.yml

@@ -15,6 +15,10 @@ defaults:
   run:
     working-directory: aws/lambda/opensearch-gha-jobs/
 
+permissions:
+  id-token: write
+  contents: read
+
 jobs:
   test:
     runs-on: ubuntu-22.04
@@ -37,7 +41,9 @@ jobs:
         with:
           python-version: '3.11'
           cache: pip
+      - name: configure aws credentials
+        uses: aws-actions/[email protected]
+        with:
+          role-to-assume: arn:aws:iam::308535385114:role/gha_workflow_opensearch-gha-jobs-lambda
+          aws-region: us-east-1
       - run: make deploy
-        env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}

.github/workflows/update-slow-tests.yml

@@ -10,6 +10,11 @@ on:
 defaults:
   run:
     working-directory: torchci
+
+permissions:
+  id-token: write
+  contents: read
+
 jobs:
   update-slow-stats:
     runs-on: ubuntu-20.04
@@ -31,10 +36,13 @@ jobs:
           user_email: '[email protected]'
           user_name: 'PyTorch Test Infra'
           commit_message: 'Updating slow tests stats'
+      - name: configure aws credentials
+        id: aws_creds
+        uses: aws-actions/[email protected]
+        with:
+          role-to-assume: arn:aws:iam::308535385114:role/gha_workflow_update-slow-tests
+          aws-region: us-east-1
       - name: Upload file to s3
         run: |
           python3 -mpip install awscli==1.27.69
           aws s3 cp "slow-tests.json" s3://ossci-metrics/slow-tests.json
-        env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}