From 4e360ab8460334b34e258e7ba5dc49b9fc12863f Mon Sep 17 00:00:00 2001 From: atalman Date: Tue, 19 Nov 2024 12:57:52 -0800 Subject: [PATCH 01/10] Try alternative to checkout test test test test test test test test fix test test test test test test test test test test test test test test test test test test test test test test test test test test test fix test --- .github/actions/binary-upload/action.yml | 57 +++++++++++++++++++ .../actions/setup-binary-builds/action.yml | 16 ++++++ .github/workflows/_binary_upload.yml | 6 +- .github/workflows/build_wheels_linux.yml | 30 ++++++++-- 4 files changed, 103 insertions(+), 6 deletions(-) create mode 100644 .github/actions/binary-upload/action.yml diff --git a/.github/actions/binary-upload/action.yml b/.github/actions/binary-upload/action.yml new file mode 100644 index 0000000000..b898c4bba6 --- /dev/null +++ b/.github/actions/binary-upload/action.yml @@ -0,0 +1,57 @@ +name: Binary upload action + +description: pull a specific docker image + +inputs: + repository: + description: 'Repository to checkout, defaults to ""' + default: '' + type: string + trigger-event: + description: "Trigger Event in caller that determines whether or not to upload" + type: string + default: '' + +runs: + using: composite + steps: + - name: Configure aws credentials (pytorch account) + if: ${{ inputs.trigger-event == 'schedule' || (inputs.trigger-event == 'push' && startsWith(github.event.ref, 'refs/heads/nightly')) }} + uses: aws-actions/configure-aws-credentials@v3 + with: + role-to-assume: arn:aws:iam::749337293305:role/gha_workflow_nightly_build_wheels + aws-region: us-east-1 + + - name: Configure aws credentials (pytorch account) + if: ${{ env.CHANNEL == 'test' && startsWith(github.event.ref, 'refs/tags/v') }} + uses: aws-actions/configure-aws-credentials@v3 + with: + role-to-assume: arn:aws:iam::749337293305:role/gha_workflow_test_build_wheels + aws-region: us-east-1 + + - name: Nightly or release RC + if: ${{ inputs.trigger-event == 'schedule' || (inputs.trigger-event == 'push' && startsWith(github.event.ref, 'refs/heads/nightly')) || (env.CHANNEL == 'test' && startsWith(github.event.ref, 'refs/tags/')) }} + shell: bash + run: | + set -ex + echo "NIGHTLY_OR_TEST=1" >> "${GITHUB_ENV}" + + - name: Upload package to pytorch.org + shell: bash + working-directory: ${{ inputs.repository }} + run: | + set -ex + + # shellcheck disable=SC1090 + source "${BUILD_ENV_FILE}" + + pip install awscli==1.32.18 + + AWS_CMD="aws s3 cp --dryrun" + if [[ "${NIGHTLY_OR_TEST:-0}" == "1" ]]; then + AWS_CMD="aws s3 cp" + fi + + for pkg in dist/*; do + ${AWS_CMD} "$pkg" "${PYTORCH_S3_BUCKET_PATH}" --acl public-read + done diff --git a/.github/actions/setup-binary-builds/action.yml b/.github/actions/setup-binary-builds/action.yml index 52fbf36a20..f21281fbbe 100644 --- a/.github/actions/setup-binary-builds/action.yml +++ b/.github/actions/setup-binary-builds/action.yml @@ -40,6 +40,11 @@ inputs: required: false type: boolean default: no + is-manylinux2-28: + description: One of the parameter used by pkg-helpers + required: false + type: boolean + default: true runs: using: composite @@ -48,15 +53,26 @@ runs: shell: bash env: REPOSITORY: ${{ inputs.repository }} + IS_MANYLINUX2_28: ${{ inputs.is-manylinux2-28 }} run: | set -euxo pipefail rm -rf "${REPOSITORY}" + - uses: actions/checkout@v4 + if: ${{ env.IS_MANYLINUX2_28 == 'true' }} with: repository: ${{ inputs.repository }} ref: ${{ inputs.ref }} submodules: ${{ inputs.submodules }} path: ${{ inputs.repository }} + + - uses: atalman/checkout-action@main + if: ${{ env.IS_MANYLINUX2_28 != 'true' }} + with: + repository: ${{ inputs.repository }} + path: ${{ inputs.repository }} + ref: ${{ inputs.ref }} + - name: Log Available Webhook Fields shell: bash run: | diff --git a/.github/workflows/_binary_upload.yml b/.github/workflows/_binary_upload.yml index ece5991aa8..604258c122 100644 --- a/.github/workflows/_binary_upload.yml +++ b/.github/workflows/_binary_upload.yml @@ -50,6 +50,8 @@ jobs: matrix: ${{ fromJSON(inputs.build-matrix) }} timeout-minutes: 30 name: ${{ matrix.build_name }} + env: + SKIP-UPLOAD: ${{ !contains(matrix.container_image, '2_28') }} steps: - uses: actions/checkout@v4 with: @@ -70,6 +72,7 @@ jobs: upload-to-base-bucket: ${{ matrix.upload_to_base_bucket }} - name: Download the artifact + if: ${{ !env.SKIP-UPLOAD }} uses: actions/download-artifact@v3 with: name: ${{ env.ARTIFACT_NAME }} @@ -97,6 +100,7 @@ jobs: echo "NIGHTLY_OR_TEST=1" >> "${GITHUB_ENV}" - name: Upload package to pytorch.org + if: ${{ !env.SKIP-UPLOAD }} shell: bash working-directory: ${{ inputs.repository }} run: | @@ -117,7 +121,7 @@ jobs: done - name: Upload package to pypi - if: ${{ env.NIGHTLY_OR_TEST == '1' && contains(inputs.upload-to-pypi, matrix.desired_cuda) }} + if: ${{ !env.SKIP-UPLOAD && env.NIGHTLY_OR_TEST == '1' && contains(inputs.upload-to-pypi, matrix.desired_cuda) }} uses: pypa/gh-action-pypi-publish@release/v1 with: user: __token__ diff --git a/.github/workflows/build_wheels_linux.yml b/.github/workflows/build_wheels_linux.yml index 2813ef1e19..01e15dd424 100644 --- a/.github/workflows/build_wheels_linux.yml +++ b/.github/workflows/build_wheels_linux.yml @@ -123,6 +123,7 @@ jobs: CU_VERSION: ${{ matrix.desired_cuda }} UPLOAD_TO_BASE_BUCKET: ${{ matrix.upload_to_base_bucket }} ARCH: ${{ inputs.architecture }} + IS_MANYLINUX2_28: ${{ contains(matrix.container_image, '2_28') }} name: ${{ matrix.build_name }} runs-on: ${{ matrix.validation_runner }} container: @@ -141,15 +142,24 @@ jobs: if [[ "${{ inputs.architecture }}" = "aarch64" ]]; then rm -rf "${RUNNER_TEMP}/*" fi - echo "::endgroup::" - - uses: actions/checkout@v4 + + - uses: actions/checkout@v3 + if: ${{ env.IS_MANYLINUX2_28 == 'true' }} with: # Support the use case where we need to checkout someone's fork repository: ${{ inputs.test-infra-repository }} ref: ${{ inputs.test-infra-ref }} path: test-infra - - name: Set linux aarch64 CI + + - uses: atalman/checkout-action@main + if: ${{ env.IS_MANYLINUX2_28 != 'true' }} + with: + repository: ${{ inputs.test-infra-repository }} + ref: ${{ inputs.test-infra-ref }} + path: test-infra + + - name: Install Miniforge if: ${{ inputs.architecture == 'aarch64' }} shell: bash -l {0} env: @@ -157,7 +167,6 @@ jobs: run: | set -euxo pipefail # TODO: Get rid of Conda, we already have all versions of PyThon one needs in the docker - ############################################################################### # Install conda # disable SSL_verify due to getting "Could not find a suitable TLS CA certificate bundle, invalid path" @@ -185,10 +194,12 @@ jobs: repository: ${{ inputs.repository }} ref: ${{ inputs.ref }} submodules: ${{ inputs.submodules }} - setup-miniconda: ${{ inputs.setup-miniconda }} + setup-miniconda: false python-version: ${{ env.PYTHON_VERSION }} cuda-version: ${{ env.CU_VERSION }} arch: ${{ env.ARCH }} + is-manylinux2-28: ${{ env.IS_MANYLINUX2_28 }} + - name: Combine Env Var and Build Env Files if: ${{ inputs.env-var-script != '' }} working-directory: ${{ inputs.repository }} @@ -281,7 +292,16 @@ jobs: ${CONDA_RUN} python "${{ inputs.repository }}/${SMOKE_TEST_SCRIPT}" fi # NB: Only upload to GitHub after passing smoke tests + + - name: Upload wheel + if: ${{ env.IS_MANYLINUX2_28 != 'true' }} + uses: ./test-infra/.github/actions/binary-upload + with: + repository: ${{ inputs.repository }} + trigger-event: ${{ inputs.trigger-event }} + - name: Upload wheel to GitHub + if: ${{ env.IS_MANYLINUX2_28 == 'true' }} continue-on-error: true uses: actions/upload-artifact@v3 with: From b2e130a804c56276736f6f1044f8c85cabbb87d5 Mon Sep 17 00:00:00 2001 From: atalman Date: Wed, 20 Nov 2024 08:18:55 -0800 Subject: [PATCH 02/10] test --- .github/workflows/build_wheels_linux.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_wheels_linux.yml b/.github/workflows/build_wheels_linux.yml index 01e15dd424..334393ae58 100644 --- a/.github/workflows/build_wheels_linux.yml +++ b/.github/workflows/build_wheels_linux.yml @@ -144,7 +144,7 @@ jobs: fi echo "::endgroup::" - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 if: ${{ env.IS_MANYLINUX2_28 == 'true' }} with: # Support the use case where we need to checkout someone's fork From 60447fd95d4ef70d18e889bb17dff1a3b685ffcd Mon Sep 17 00:00:00 2001 From: atalman Date: Wed, 20 Nov 2024 11:46:41 -0800 Subject: [PATCH 03/10] use_same_checkout --- .github/workflows/build_wheels_linux.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.github/workflows/build_wheels_linux.yml b/.github/workflows/build_wheels_linux.yml index 334393ae58..0018295e47 100644 --- a/.github/workflows/build_wheels_linux.yml +++ b/.github/workflows/build_wheels_linux.yml @@ -144,16 +144,7 @@ jobs: fi echo "::endgroup::" - - uses: actions/checkout@v4 - if: ${{ env.IS_MANYLINUX2_28 == 'true' }} - with: - # Support the use case where we need to checkout someone's fork - repository: ${{ inputs.test-infra-repository }} - ref: ${{ inputs.test-infra-ref }} - path: test-infra - - uses: atalman/checkout-action@main - if: ${{ env.IS_MANYLINUX2_28 != 'true' }} with: repository: ${{ inputs.test-infra-repository }} ref: ${{ inputs.test-infra-ref }} From 72c77b0f9f0aab7ff4c264d09139e7f8ba418614 Mon Sep 17 00:00:00 2001 From: atalman Date: Wed, 20 Nov 2024 11:58:17 -0800 Subject: [PATCH 04/10] refactor_setup_binary_builds --- .github/actions/setup-binary-builds/action.yml | 8 ++++---- .github/workflows/build_wheels_linux.yml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/actions/setup-binary-builds/action.yml b/.github/actions/setup-binary-builds/action.yml index f21281fbbe..7f7b256b5a 100644 --- a/.github/actions/setup-binary-builds/action.yml +++ b/.github/actions/setup-binary-builds/action.yml @@ -40,7 +40,7 @@ inputs: required: false type: boolean default: no - is-manylinux2-28: + use-github-checkout: description: One of the parameter used by pkg-helpers required: false type: boolean @@ -53,13 +53,13 @@ runs: shell: bash env: REPOSITORY: ${{ inputs.repository }} - IS_MANYLINUX2_28: ${{ inputs.is-manylinux2-28 }} + USE_GITHUB_CHECKOUT: ${{ inputs.use-github-checkout }} run: | set -euxo pipefail rm -rf "${REPOSITORY}" - uses: actions/checkout@v4 - if: ${{ env.IS_MANYLINUX2_28 == 'true' }} + if: ${{ env.USE_GITHUB_CHECKOUT == 'true' }} with: repository: ${{ inputs.repository }} ref: ${{ inputs.ref }} @@ -67,7 +67,7 @@ runs: path: ${{ inputs.repository }} - uses: atalman/checkout-action@main - if: ${{ env.IS_MANYLINUX2_28 != 'true' }} + if: ${{ env.USE_GITHUB_CHECKOUT != 'true' }} with: repository: ${{ inputs.repository }} path: ${{ inputs.repository }} diff --git a/.github/workflows/build_wheels_linux.yml b/.github/workflows/build_wheels_linux.yml index 0018295e47..93195e205e 100644 --- a/.github/workflows/build_wheels_linux.yml +++ b/.github/workflows/build_wheels_linux.yml @@ -189,7 +189,7 @@ jobs: python-version: ${{ env.PYTHON_VERSION }} cuda-version: ${{ env.CU_VERSION }} arch: ${{ env.ARCH }} - is-manylinux2-28: ${{ env.IS_MANYLINUX2_28 }} + use-github-checkout: ${{ env.IS_MANYLINUX2_28 }} - name: Combine Env Var and Build Env Files if: ${{ inputs.env-var-script != '' }} From fccbba707e09b3f488a66c2818c42d13b226c243 Mon Sep 17 00:00:00 2001 From: atalman Date: Wed, 20 Nov 2024 15:18:33 -0800 Subject: [PATCH 05/10] test --- .github/actions/setup-binary-builds/action.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/actions/setup-binary-builds/action.yml b/.github/actions/setup-binary-builds/action.yml index 7f7b256b5a..d5d87c2cb1 100644 --- a/.github/actions/setup-binary-builds/action.yml +++ b/.github/actions/setup-binary-builds/action.yml @@ -53,13 +53,12 @@ runs: shell: bash env: REPOSITORY: ${{ inputs.repository }} - USE_GITHUB_CHECKOUT: ${{ inputs.use-github-checkout }} run: | set -euxo pipefail rm -rf "${REPOSITORY}" - uses: actions/checkout@v4 - if: ${{ env.USE_GITHUB_CHECKOUT == 'true' }} + if: ${{ inputs.use-github-checkout }} with: repository: ${{ inputs.repository }} ref: ${{ inputs.ref }} @@ -67,7 +66,7 @@ runs: path: ${{ inputs.repository }} - uses: atalman/checkout-action@main - if: ${{ env.USE_GITHUB_CHECKOUT != 'true' }} + if: ${{ !inputs.use-github-checkout }} with: repository: ${{ inputs.repository }} path: ${{ inputs.repository }} From d62780f703e93cdb4059016fb903e57106b6cbe1 Mon Sep 17 00:00:00 2001 From: atalman Date: Wed, 20 Nov 2024 15:32:31 -0800 Subject: [PATCH 06/10] fix_input+param --- .github/actions/setup-binary-builds/action.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/actions/setup-binary-builds/action.yml b/.github/actions/setup-binary-builds/action.yml index d5d87c2cb1..80cbbcf1ad 100644 --- a/.github/actions/setup-binary-builds/action.yml +++ b/.github/actions/setup-binary-builds/action.yml @@ -53,12 +53,13 @@ runs: shell: bash env: REPOSITORY: ${{ inputs.repository }} + USE_GITHUB_CHECKOUT: ${{ inputs.use-github-checkout }} run: | set -euxo pipefail rm -rf "${REPOSITORY}" - uses: actions/checkout@v4 - if: ${{ inputs.use-github-checkout }} + if: ${{ env.USE_GITHUB_CHECKOUT == 'true' }} with: repository: ${{ inputs.repository }} ref: ${{ inputs.ref }} @@ -66,7 +67,7 @@ runs: path: ${{ inputs.repository }} - uses: atalman/checkout-action@main - if: ${{ !inputs.use-github-checkout }} + if: ${{ env.USE_GITHUB_CHECKOUT == 'false' }} with: repository: ${{ inputs.repository }} path: ${{ inputs.repository }} From a7303de71aa7e4604cb546b5196c9c110aca2355 Mon Sep 17 00:00:00 2001 From: atalman Date: Wed, 20 Nov 2024 15:43:17 -0800 Subject: [PATCH 07/10] typo --- .github/actions/setup-binary-builds/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/setup-binary-builds/action.yml b/.github/actions/setup-binary-builds/action.yml index 80cbbcf1ad..7f7b256b5a 100644 --- a/.github/actions/setup-binary-builds/action.yml +++ b/.github/actions/setup-binary-builds/action.yml @@ -67,7 +67,7 @@ runs: path: ${{ inputs.repository }} - uses: atalman/checkout-action@main - if: ${{ env.USE_GITHUB_CHECKOUT == 'false' }} + if: ${{ env.USE_GITHUB_CHECKOUT != 'true' }} with: repository: ${{ inputs.repository }} path: ${{ inputs.repository }} From bfc9a814e8359cefbff094b26ea4b08d7aaee194 Mon Sep 17 00:00:00 2001 From: atalman Date: Wed, 20 Nov 2024 17:12:17 -0800 Subject: [PATCH 08/10] fix_boolean_param --- .github/actions/setup-binary-builds/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/setup-binary-builds/action.yml b/.github/actions/setup-binary-builds/action.yml index 7f7b256b5a..2b687ea399 100644 --- a/.github/actions/setup-binary-builds/action.yml +++ b/.github/actions/setup-binary-builds/action.yml @@ -59,7 +59,7 @@ runs: rm -rf "${REPOSITORY}" - uses: actions/checkout@v4 - if: ${{ env.USE_GITHUB_CHECKOUT == 'true' }} + if: ${{ fromJSON(env.USE_GITHUB_CHECKOUT) }} with: repository: ${{ inputs.repository }} ref: ${{ inputs.ref }} @@ -67,7 +67,7 @@ runs: path: ${{ inputs.repository }} - uses: atalman/checkout-action@main - if: ${{ env.USE_GITHUB_CHECKOUT != 'true' }} + if: ${{ !fromJSON(env.USE_GITHUB_CHECKOUT) }} with: repository: ${{ inputs.repository }} path: ${{ inputs.repository }} From 10fb3fa9ceb4231b7e7a300e4eb65ffa3254654d Mon Sep 17 00:00:00 2001 From: atalman Date: Wed, 20 Nov 2024 17:14:53 -0800 Subject: [PATCH 09/10] fix_boolean_passing --- .github/actions/setup-binary-builds/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/setup-binary-builds/action.yml b/.github/actions/setup-binary-builds/action.yml index 2b687ea399..d0c6ab492a 100644 --- a/.github/actions/setup-binary-builds/action.yml +++ b/.github/actions/setup-binary-builds/action.yml @@ -43,8 +43,8 @@ inputs: use-github-checkout: description: One of the parameter used by pkg-helpers required: false - type: boolean - default: true + type: string + default: 'true' runs: using: composite From 14008c4b941085f1eef78801378a6cfa379b5fa0 Mon Sep 17 00:00:00 2001 From: atalman Date: Wed, 20 Nov 2024 17:18:42 -0800 Subject: [PATCH 10/10] booleans --- .github/actions/setup-binary-builds/action.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/actions/setup-binary-builds/action.yml b/.github/actions/setup-binary-builds/action.yml index d0c6ab492a..5123929db6 100644 --- a/.github/actions/setup-binary-builds/action.yml +++ b/.github/actions/setup-binary-builds/action.yml @@ -43,8 +43,8 @@ inputs: use-github-checkout: description: One of the parameter used by pkg-helpers required: false - type: string - default: 'true' + type: boolean + default: true runs: using: composite @@ -53,13 +53,12 @@ runs: shell: bash env: REPOSITORY: ${{ inputs.repository }} - USE_GITHUB_CHECKOUT: ${{ inputs.use-github-checkout }} run: | set -euxo pipefail rm -rf "${REPOSITORY}" - uses: actions/checkout@v4 - if: ${{ fromJSON(env.USE_GITHUB_CHECKOUT) }} + if: ${{ inputs.use-github-checkout == 'true' }} with: repository: ${{ inputs.repository }} ref: ${{ inputs.ref }} @@ -67,7 +66,7 @@ runs: path: ${{ inputs.repository }} - uses: atalman/checkout-action@main - if: ${{ !fromJSON(env.USE_GITHUB_CHECKOUT) }} + if: ${{ inputs.use-github-checkout == 'false' }} with: repository: ${{ inputs.repository }} path: ${{ inputs.repository }}