From 99db1eb062c6ebf1743a6f5859128e9a9c5275de Mon Sep 17 00:00:00 2001 From: Jean Schmidt <4520845+jeanschmidt@users.noreply.github.com> Date: Wed, 6 Dec 2023 17:03:07 +0100 Subject: [PATCH] Revert "Revert "DO NOT REVERT - Updating workflows to gather AWS authentication via OIDC" (#4785)" This reverts commit 3a745d27bd9343af5ea75033480ef2cae086ad20. --- .github/workflows/backfill-workflow-job.yml | 12 +++++++-- .github/workflows/build_wheels_linux.yml | 19 +++++++------- .github/workflows/build_wheels_macos.yml | 19 +++++++------- .github/workflows/build_wheels_windows.yml | 19 +++++++------- .github/workflows/clang-tidy-linux.yml | 15 +++++++++-- .github/workflows/clang-tidy-macos.yml | 26 ++++++++++++++++--- .../workflows/github-status-test-lambda.yml | 12 ++++++--- .../lambda_github_status_webhook_handler.yml | 15 +++++++++-- .github/workflows/log-classifier-lambda.yml | 12 ++++++--- .../workflows/opensearch-gha-jobs-lambda.yml | 12 ++++++--- .github/workflows/update-slow-tests.yml | 14 +++++++--- .../workflows/update_ci_wait_time_metric.yml | 14 +++++++--- .github/workflows/update_disabled_tests.yml | 13 +++++++--- .github/workflows/upload-tutorials-stats.yml | 16 +++++++----- 14 files changed, 154 insertions(+), 64 deletions(-) diff --git a/.github/workflows/backfill-workflow-job.yml b/.github/workflows/backfill-workflow-job.yml index 78dcaf5a64..23f94c9818 100644 --- a/.github/workflows/backfill-workflow-job.yml +++ b/.github/workflows/backfill-workflow-job.yml @@ -8,16 +8,24 @@ on: defaults: run: working-directory: torchci + +permissions: + id-token: write + contents: read + jobs: backfill-rockset: runs-on: ubuntu-20.04 steps: + - name: configure aws credentials + uses: aws-actions/configure-aws-credentials@v1.7.0 + with: + role-to-assume: arn:aws:iam::308535385114:role/gha_workflow_backfill-workflow-job + aws-region: us-east-1 - uses: actions/checkout@v3 - run: yarn install --frozen-lockfile - run: yarn node scripts/backfillJobs.mjs env: ROCKSET_API_KEY: ${{ secrets.ROCKSET_API_KEY }} - OUR_AWS_ACCESS_KEY_ID: ${{ secrets.OUR_AWS_ACCESS_KEY_ID }} - OUR_AWS_SECRET_ACCESS_KEY: ${{ secrets.OUR_AWS_SECRET_ACCESS_KEY }} APP_ID: ${{ secrets.APP_ID }} PRIVATE_KEY: ${{ secrets.PRIVATE_KEY }} diff --git a/.github/workflows/build_wheels_linux.yml b/.github/workflows/build_wheels_linux.yml index e241b40688..bb127bb76f 100644 --- a/.github/workflows/build_wheels_linux.yml +++ b/.github/workflows/build_wheels_linux.yml @@ -65,13 +65,10 @@ on: required: false type: boolean default: true - secrets: - AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID: - description: "AWS Access Key passed from caller workflow" - required: false - AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY: - description: "AWS Secret Access Ket passed from caller workflow" - required: false + +permissions: + id-token: write + contents: read jobs: build: @@ -122,6 +119,11 @@ jobs: repository: "pytorch/builder" ref: "main" path: builder + - name: configure aws credentials + uses: aws-actions/configure-aws-credentials@v1.7.0 + with: + role-to-assume: arn:aws:iam::308535385114:role/gha_workflow_build_wheels_linux + aws-region: us-east-1 - name: Set linux aarch64 CI if: ${{ inputs.architecture == 'aarch64' }} shell: bash -l {0} @@ -228,9 +230,6 @@ jobs: if: ${{ (inputs.trigger-event == 'push' && startsWith(github.event.ref, 'refs/heads/nightly')) || (env.CHANNEL == 'test' && startsWith(github.event.ref, 'refs/tags/')) }} shell: bash -l {0} working-directory: ${{ inputs.repository }} - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }} run: | set -euxo pipefail source "${BUILD_ENV_FILE}" diff --git a/.github/workflows/build_wheels_macos.yml b/.github/workflows/build_wheels_macos.yml index fbb661c2fd..b1bda8e62f 100644 --- a/.github/workflows/build_wheels_macos.yml +++ b/.github/workflows/build_wheels_macos.yml @@ -59,13 +59,10 @@ on: description: "The key created when saving a cache and the key used to search for a cache." default: "" type: string - secrets: - AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID: - description: "AWS Access Key passed from caller workflow" - required: false - AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY: - description: "AWS Secret Access Ket passed from caller workflow" - required: false + +permissions: + id-token: write + contents: read jobs: build: @@ -97,6 +94,11 @@ jobs: repository: ${{ inputs.test-infra-repository }} ref: ${{ inputs.test-infra-ref }} path: test-infra + - name: configure aws credentials + uses: aws-actions/configure-aws-credentials@v1.7.0 + with: + role-to-assume: arn:aws:iam::308535385114:role/gha_workflow_build_wheels_macos + aws-region: us-east-1 - uses: ./test-infra/.github/actions/set-channel - uses: ./test-infra/.github/actions/setup-binary-builds with: @@ -202,9 +204,6 @@ jobs: if: ${{ (inputs.trigger-event == 'push' && startsWith(github.event.ref, 'refs/heads/nightly')) || (env.CHANNEL == 'test' && startsWith(github.event.ref, 'refs/tags/')) }} shell: bash -l {0} working-directory: ${{ inputs.repository }} - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }} run: | set -euxo pipefail # shellcheck disable=SC1090 diff --git a/.github/workflows/build_wheels_windows.yml b/.github/workflows/build_wheels_windows.yml index dd834ababe..f59d96b2e6 100644 --- a/.github/workflows/build_wheels_windows.yml +++ b/.github/workflows/build_wheels_windows.yml @@ -59,13 +59,10 @@ on: description: "The key created when saving a cache and the key used to search for a cache." default: "" type: string - secrets: - AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID: - description: "AWS Access Key passed from caller workflow" - required: false - AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY: - description: "AWS Secret Access Ket passed from caller workflow" - required: false + +permissions: + id-token: write + contents: read jobs: build: @@ -188,13 +185,15 @@ jobs: echo "${{ inputs.repository }}/${SMOKE_TEST_SCRIPT} found" ${CONDA_RUN} python "${{ inputs.repository }}/${SMOKE_TEST_SCRIPT}" fi + - name: configure aws credentials + uses: aws-actions/configure-aws-credentials@v1.7.0 + with: + role-to-assume: arn:aws:iam::308535385114:role/gha_workflow_build_wheels_windows + aws-region: us-east-1 - name: Upload package to pytorch.org if: ${{ (inputs.trigger-event == 'push' && startsWith(github.event.ref, 'refs/heads/nightly')) || (env.CHANNEL == 'test' && startsWith(github.event.ref, 'refs/tags/')) }} shell: bash -l {0} working-directory: ${{ inputs.repository }} - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }} run: | source "${BUILD_ENV_FILE}" ${CONDA_RUN} pip install awscli diff --git a/.github/workflows/clang-tidy-linux.yml b/.github/workflows/clang-tidy-linux.yml index 0e11b03e5e..3b07073259 100644 --- a/.github/workflows/clang-tidy-linux.yml +++ b/.github/workflows/clang-tidy-linux.yml @@ -14,6 +14,10 @@ on: - '!tools/clang-tidy-checks/README.md' - '.github/workflows/clang-tidy-linux.yml' +permissions: + id-token: write + contents: read + jobs: build: runs-on: linux.12xlarge @@ -36,6 +40,13 @@ jobs: image_id=$(docker create ghcr.io/pytorch/cilint-clang-tidy:"$GITHUB_SHA") docker cp "$image_id":/clang-tidy-checks/build/bin/clang-tidy ./clang-tidy docker rm -v "$image_id" + - name: configure aws credentials + uses: aws-actions/configure-aws-credentials@v1.7.0 + id: aws_creds + with: + role-to-assume: arn:aws:iam::308535385114:role/gha_workflow_clang-tidy-linux + aws-region: us-east-1 + output-credentials: true - uses: driazati/upload-artifact-s3@50adbe4ef0b6d9221df25c18c5fc528dfcb7c3f8 name: Publish binary if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} @@ -45,8 +56,8 @@ jobs: s3-prefix: linux64/17.0.6 s3-bucket: oss-clang-format path: clang-tidy - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws_access_key_id: ${{ steps.aws_creds.outputs.aws-access-key-id }} + aws_secret_access_key: ${{ steps.aws_creds.outputs.aws-secret-access-key }} concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} diff --git a/.github/workflows/clang-tidy-macos.yml b/.github/workflows/clang-tidy-macos.yml index c34264a173..50c4643072 100644 --- a/.github/workflows/clang-tidy-macos.yml +++ b/.github/workflows/clang-tidy-macos.yml @@ -16,6 +16,10 @@ on: - '!tools/clang-tidy-checks/README.md' - '.github/workflows/clang-tidy-macos.yml' +permissions: + id-token: write + contents: read + jobs: build-Intel: runs-on: macos-12-xl @@ -38,6 +42,13 @@ jobs: export PATH ./setup.sh + - name: configure aws credentials + id: aws_creds + uses: aws-actions/configure-aws-credentials@v1.7.0 + with: + role-to-assume: arn:aws:iam::308535385114:role/gha_workflow_clang-tidy-macos + aws-region: us-east-1 + output-credentials: true - uses: driazati/upload-artifact-s3@50adbe4ef0b6d9221df25c18c5fc528dfcb7c3f8 name: Publish binary if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} @@ -47,8 +58,8 @@ jobs: s3-prefix: macos-i386/17.0.6 s3-bucket: oss-clang-format path: tools/clang-tidy-checks/llvm-project/build/bin/clang-tidy - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws_access_key_id: ${{ steps.aws_creds.outputs.aws-access-key-id }} + aws_secret_access_key: ${{ steps.aws_creds.outputs.aws-secret-access-key }} build-M1: runs-on: macos-m1-12 steps: @@ -67,6 +78,13 @@ jobs: export PATH ./setup.sh + - name: configure aws credentials + id: aws_creds + uses: aws-actions/configure-aws-credentials@v1.7.0 + with: + role-to-assume: arn:aws:iam::308535385114:role/gha_workflow_clang-tidy-macos + aws-region: us-east-1 + output-credentials: true - uses: driazati/upload-artifact-s3@50adbe4ef0b6d9221df25c18c5fc528dfcb7c3f8 name: Publish binary if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} @@ -76,8 +94,8 @@ jobs: s3-prefix: macos-arm/17.0.6 s3-bucket: oss-clang-format path: tools/clang-tidy-checks/llvm-project/build/bin/clang-tidy - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws_access_key_id: ${{ steps.aws_creds.outputs.aws-access-key-id }} + aws_secret_access_key: ${{ steps.aws_creds.outputs.aws-secret-access-key }} concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} diff --git a/.github/workflows/github-status-test-lambda.yml b/.github/workflows/github-status-test-lambda.yml index 7340b77597..27ff888995 100644 --- a/.github/workflows/github-status-test-lambda.yml +++ b/.github/workflows/github-status-test-lambda.yml @@ -16,6 +16,10 @@ defaults: run: working-directory: aws/lambda/github-status-test/ +permissions: + id-token: write + contents: read + jobs: deploy: runs-on: ubuntu-latest @@ -36,13 +40,15 @@ jobs: command: | set -eux python3 -m pip install awscli + - name: configure aws credentials + uses: aws-actions/configure-aws-credentials@v1.7.0 + with: + role-to-assume: arn:aws:iam::308535385114:role/gha_workflow_github-status-test-lambda + aws-region: us-east-1 - name: Prepare package run: | make prepare - name: Deploy the lambda. (This only makes the lambda ready. Manual verification steps are still required to get it into prod because of the lack of automated testing here. Checkout aws/lambda/github-status-test/github-status-test/README.md to see how to get the deployed lambda to prod) if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} run: | make deploy diff --git a/.github/workflows/lambda_github_status_webhook_handler.yml b/.github/workflows/lambda_github_status_webhook_handler.yml index bc7ec0735e..83913dc543 100644 --- a/.github/workflows/lambda_github_status_webhook_handler.yml +++ b/.github/workflows/lambda_github_status_webhook_handler.yml @@ -8,17 +8,28 @@ on: - '.github/workflows/lambda_github_status_webhook_handler.yml' - 'aws/lambda/github-status-webhook-handler/**' +permissions: + id-token: write + contents: read + jobs: deploy: runs-on: ubuntu-20.04 steps: - name: Checkout uses: actions/checkout@v3 + - name: configure aws credentials + id: aws_creds + uses: aws-actions/configure-aws-credentials@v1.7.0 + with: + role-to-assume: arn:aws:iam::308535385114:role/gha_workflow_lambda_github_status_webhook_handler + aws-region: us-east-1 + output-credentials: true - name: Deploy uses: appleboy/lambda-action@1e05c1377056f21ebb2ce69b910bc16b943c2a66 with: - aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws_access_key_id: ${{ steps.aws_creds.outputs.aws-access-key-id }} + aws_secret_access_key: ${{ steps.aws_creds.outputs.aws-secret-access-key }} aws_region: us-east-1 function_name: github-status-webhook-handler source: aws/lambda/github-status-webhook-handler/lambda_function.py diff --git a/.github/workflows/log-classifier-lambda.yml b/.github/workflows/log-classifier-lambda.yml index a4255e9f7c..19a64b3169 100644 --- a/.github/workflows/log-classifier-lambda.yml +++ b/.github/workflows/log-classifier-lambda.yml @@ -15,6 +15,10 @@ defaults: run: working-directory: aws/lambda/log-classifier/ +permissions: + id-token: write + contents: read + jobs: test: runs-on: ubuntu-20.04 @@ -29,9 +33,11 @@ jobs: steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v2 + - name: configure aws credentials + uses: aws-actions/configure-aws-credentials@v1.7.0 + with: + role-to-assume: arn:aws:iam::308535385114:role/gha_workflow_log-classifier-lambda + aws-region: us-east-1 - run: pip3 install cargo-lambda - run: cargo lambda build --release - run: cargo lambda deploy - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/opensearch-gha-jobs-lambda.yml b/.github/workflows/opensearch-gha-jobs-lambda.yml index c2ed1f9345..dc1de14981 100644 --- a/.github/workflows/opensearch-gha-jobs-lambda.yml +++ b/.github/workflows/opensearch-gha-jobs-lambda.yml @@ -15,6 +15,10 @@ defaults: run: working-directory: aws/lambda/opensearch-gha-jobs/ +permissions: + id-token: write + contents: read + jobs: test: runs-on: ubuntu-22.04 @@ -37,7 +41,9 @@ jobs: with: python-version: '3.11' cache: pip + - name: configure aws credentials + uses: aws-actions/configure-aws-credentials@v1.7.0 + with: + role-to-assume: arn:aws:iam::308535385114:role/gha_workflow_opensearch-gha-jobs-lambda + aws-region: us-east-1 - run: make deploy - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/update-slow-tests.yml b/.github/workflows/update-slow-tests.yml index 4fe00a7e85..bea9769b79 100644 --- a/.github/workflows/update-slow-tests.yml +++ b/.github/workflows/update-slow-tests.yml @@ -10,6 +10,11 @@ on: defaults: run: working-directory: torchci + +permissions: + id-token: write + contents: read + jobs: update-slow-stats: runs-on: ubuntu-20.04 @@ -31,10 +36,13 @@ jobs: user_email: 'test-infra@pytorch.org' user_name: 'PyTorch Test Infra' commit_message: 'Updating slow tests stats' + - name: configure aws credentials + id: aws_creds + uses: aws-actions/configure-aws-credentials@v1.7.0 + with: + role-to-assume: arn:aws:iam::308535385114:role/gha_workflow_update-slow-tests + aws-region: us-east-1 - name: Upload file to s3 run: | python3 -mpip install awscli==1.27.69 aws s3 cp "slow-tests.json" s3://ossci-metrics/slow-tests.json - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/update_ci_wait_time_metric.yml b/.github/workflows/update_ci_wait_time_metric.yml index 6184ce695e..3a707e2a27 100644 --- a/.github/workflows/update_ci_wait_time_metric.yml +++ b/.github/workflows/update_ci_wait_time_metric.yml @@ -3,9 +3,13 @@ on: schedule: # Update the indices every day at 5am - cron: "0 5 * * *" - # Enable triggering this job manually using the API as well + # Enable triggering this job manually using the API as well workflow_dispatch: +permissions: + id-token: write + contents: read + jobs: update-kpi: runs-on: ubuntu-22.04 @@ -27,10 +31,14 @@ jobs: pip3 install boto3==1.19.12 pip3 install pandas==1.5 + - name: configure aws credentials + uses: aws-actions/configure-aws-credentials@v1.7.0 + with: + role-to-assume: arn:aws:iam::308535385114:role/gha_workflow_update_ci_wait_time_metric + aws-region: us-east-1 + - name: Compute kpi and upload to RDS env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} ROCKSET_API_KEY: ${{ secrets.ROCKSET_API_KEY }} run: | python3 .github/scripts/compute_and_upload_ci_wait_time_metric.py diff --git a/.github/workflows/update_disabled_tests.yml b/.github/workflows/update_disabled_tests.yml index 00e44fb093..5accefa8aa 100644 --- a/.github/workflows/update_disabled_tests.yml +++ b/.github/workflows/update_disabled_tests.yml @@ -10,6 +10,10 @@ on: paths: - .github/scripts/update_disabled_issues.py +permissions: + id-token: write + contents: read + jobs: update-disabled-tests: runs-on: ubuntu-latest @@ -83,6 +87,12 @@ jobs: user_name: "Pytorch Test Infra" commit_message: "Updating unstable jobs" + - name: configure aws credentials + uses: aws-actions/configure-aws-credentials@v1.7.0 + with: + role-to-assume: arn:aws:iam::308535385114:role/gha_workflow_update_disabled_tests + aws-region: us-east-1 + - name: Upload file to s3 if: github.event_name != 'pull_request' run: | @@ -91,6 +101,3 @@ jobs: aws s3 cp disabled-tests-condensed.json s3://ossci-metrics/disabled-tests-condensed.json aws s3 cp disabled-jobs.json s3://ossci-metrics/disabled-jobs.json aws s3 cp unstable-jobs.json s3://ossci-metrics/unstable-jobs.json - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/upload-tutorials-stats.yml b/.github/workflows/upload-tutorials-stats.yml index f9332eabb7..b74fc6b794 100644 --- a/.github/workflows/upload-tutorials-stats.yml +++ b/.github/workflows/upload-tutorials-stats.yml @@ -4,7 +4,7 @@ name: Upload tutorials stat # Controls when the workflow will run on: - pull_request: + pull_request: paths: - .github/workflows/upload-tutorials-stats.yml - .github/scripts/get_tutorials_stats.py @@ -14,6 +14,10 @@ on: # Allows you to run this workflow manually from the Actions tab workflow_dispatch: +permissions: + id-token: write + contents: read + jobs: get-tutorials-stats: if: ${{ github.repository == 'pytorch/test-infra' }} @@ -32,12 +36,12 @@ jobs: - name: Install dependencies run: | python3 -m pip install boto3==1.26.69 typing==3.7.4.3 - + - name: configure aws credentials + uses: aws-actions/configure-aws-credentials@v1.7.0 + with: + role-to-assume: arn:aws:iam::308535385114:role/gha_workflow_upload-tutorials-stats + aws-region: us-east-1 - name: Run the script - env: - AWS_DEFAULT_REGION: us-east-1 - AWS_ACCESS_KEY_ID: ${{ secrets.TORCHCI_TUTORIAL_DYNAMODB_READ_WRITE_AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.TORCHCI_TUTORIAL_DYNAMODB_READ_WRITE_AWS_SECRET_ACCESS_KEY }} run: | set -x python3 test-infra/.github/scripts/get_tutorials_stats.py