From ac26b29e3651bf54a588084820b8bf9256c7b61d Mon Sep 17 00:00:00 2001 From: Huy Do Date: Sat, 14 Sep 2024 08:37:38 -0700 Subject: [PATCH] Add option to upload Nova artifacts to S3 directly (#5668) For the context, Nova jobs upload the artifacts to GitHub because this works on all runners including non-AWS ones like GitHub runners. However, there is a scale issue when trying to upload large artifacts (GB in size) to GitHub that the files are too big, so the upload could either: * Timing out https://github.com/pytorch/executorch/actions/runs/10858058150/job/30136354800 * or just fail in the middle of the upload https://github.com/pytorch/executorch/actions/runs/10856291106/job/30132545832 To address this, I add the option to upload to S3 gha-artifacts bucket here which is direct and more efficient. --- .github/workflows/linux_job.yml | 23 ++++++++++++++++++++++- .github/workflows/macos_job.yml | 23 ++++++++++++++++++++++- .github/workflows/test_linux_job.yml | 10 ++++++++++ .github/workflows/test_macos_job.yml | 10 ++++++++++ .github/workflows/test_windows_job.yml | 10 ++++++++++ .github/workflows/windows_job.yml | 23 ++++++++++++++++++++++- 6 files changed, 96 insertions(+), 3 deletions(-) diff --git a/.github/workflows/linux_job.yml b/.github/workflows/linux_job.yml index 86badcc0a4..396a64a979 100644 --- a/.github/workflows/linux_job.yml +++ b/.github/workflows/linux_job.yml @@ -21,6 +21,13 @@ on: under dist/ and any files under artifacts-to-be-uploaded/ will be uploaded default: '' type: string + upload-artifact-to-s3: + description: | + Upload the artifact to S3 instead of GitHub. This is used for large artifacts like + exported model + required: false + default: false + type: boolean download-artifact: description: 'Name to download artifacts to ${RUNNER_ARTIFACT_DIR}' default: '' @@ -292,14 +299,28 @@ jobs: fi echo "upload-docs=${upload_docs}" >> "${GITHUB_OUTPUT}" + # NB: Keep this for compatibility with existing jobs and also keep in mind that only + # our AWS runners have access to S3 - name: Upload artifacts to GitHub (if any) uses: actions/upload-artifact@v3 - if: ${{ always() && inputs.upload-artifact != '' }} + if: ${{ always() && inputs.upload-artifact != '' && !inputs.upload-artifact-to-s3 }} with: name: ${{ inputs.upload-artifact }} path: ${{ runner.temp }}/artifacts/ if-no-files-found: ${{ steps.check-artifacts.outputs.if-no-files-found }} + # NB: This only works with our AWS runners + - name: Upload artifacts to S3 (if any) + uses: seemethere/upload-artifact-s3@v5 + if: ${{ always() && inputs.upload-artifact != '' && inputs.upload-artifact-to-s3 }} + with: + retention-days: 14 + s3-bucket: gha-artifacts + s3-prefix: | + ${{ env.REPOSITORY }}/${{ github.run_id }}/artifacts + if-no-files-found: ${{ steps.check-artifacts.outputs.if-no-files-found }} + path: ${{ runner.temp }}/artifacts/ + - name: Upload documentation to S3 (if any) uses: seemethere/upload-artifact-s3@v5 if: ${{ steps.check-artifacts.outputs.upload-docs == 1 && github.event.pull_request.number != '' }} diff --git a/.github/workflows/macos_job.yml b/.github/workflows/macos_job.yml index 546322aa2c..7157b2fb76 100644 --- a/.github/workflows/macos_job.yml +++ b/.github/workflows/macos_job.yml @@ -21,6 +21,13 @@ on: under dist/ and any files under artifacts-to-be-uploaded/ will be uploaded default: "" type: string + upload-artifact-to-s3: + description: | + Upload the artifact to S3 instead of GitHub. This is used for large artifacts like + exported model + required: false + default: false + type: boolean download-artifact: description: 'Name to download artifacts to ${RUNNER_ARTIFACT_DIR}' default: "" @@ -183,14 +190,28 @@ jobs: # Set to fail upload step if there are no files for upload and expected files for upload echo 'if-no-files-found=error' >> "${GITHUB_OUTPUT}" + # NB: Keep this for compatibility with existing jobs and also keep in mind that only + # our AWS runners have access to S3 - name: Upload artifacts to GitHub (if any) uses: actions/upload-artifact@v3 - if: ${{ always() && inputs.upload-artifact != '' }} + if: ${{ always() && inputs.upload-artifact != '' && !inputs.upload-artifact-to-s3 }} with: name: ${{ inputs.upload-artifact }} path: ${{ runner.temp }}/artifacts/ if-no-files-found: ${{ steps.check-artifacts.outputs.if-no-files-found }} + # NB: This only works with our AWS runners + - name: Upload artifacts to S3 (if any) + uses: seemethere/upload-artifact-s3@v5 + if: ${{ always() && inputs.upload-artifact != '' && inputs.upload-artifact-to-s3 }} + with: + retention-days: 14 + s3-bucket: gha-artifacts + s3-prefix: | + ${{ env.REPOSITORY }}/${{ github.run_id }}/artifacts + if-no-files-found: ${{ steps.check-artifacts.outputs.if-no-files-found }} + path: ${{ runner.temp }}/artifacts/ + - name: Clean up disk space if: always() continue-on-error: true diff --git a/.github/workflows/test_linux_job.yml b/.github/workflows/test_linux_job.yml index 786b3d2919..3021187f10 100644 --- a/.github/workflows/test_linux_job.yml +++ b/.github/workflows/test_linux_job.yml @@ -93,6 +93,16 @@ jobs: upload-artifact: my-cool-artifact script: | echo "hello" > "${RUNNER_ARTIFACT_DIR}/cool_beans" + test-upload-artifact-s3: + uses: ./.github/workflows/linux_job.yml + with: + runner: linux.2xlarge + test-infra-repository: ${{ github.repository }} + test-infra-ref: ${{ github.ref }} + upload-artifact: my-cool-artifact + upload-artifact-to-s3: true + script: | + echo "hello" > "${RUNNER_ARTIFACT_DIR}/cool_beans" test-download-artifact: needs: test-upload-artifact uses: ./.github/workflows/linux_job.yml diff --git a/.github/workflows/test_macos_job.yml b/.github/workflows/test_macos_job.yml index 1378135acf..61a6b6fc94 100644 --- a/.github/workflows/test_macos_job.yml +++ b/.github/workflows/test_macos_job.yml @@ -78,6 +78,16 @@ jobs: upload-artifact: my-cool-artifact script: | echo "hello" > "${RUNNER_ARTIFACT_DIR}/cool_beans" + test-upload-artifact-s3: + uses: ./.github/workflows/macos_job.yml + with: + runner: macos-m1-stable + test-infra-repository: ${{ github.repository }} + test-infra-ref: ${{ github.ref }} + upload-artifact: my-cool-artifact + upload-artifact-to-s3: true + script: | + echo "hello" > "${RUNNER_ARTIFACT_DIR}/cool_beans" test-download-artifact: needs: test-upload-artifact uses: ./.github/workflows/macos_job.yml diff --git a/.github/workflows/test_windows_job.yml b/.github/workflows/test_windows_job.yml index df9f8f0edb..4bb8935402 100644 --- a/.github/workflows/test_windows_job.yml +++ b/.github/workflows/test_windows_job.yml @@ -46,6 +46,16 @@ jobs: upload-artifact: my-cool-artifact script: | echo "hello" > "${RUNNER_ARTIFACT_DIR}/cool_beans" + test-upload-artifact-s3: + uses: ./.github/workflows/windows_job.yml + with: + runner: windows.4xlarge + test-infra-repository: ${{ github.repository }} + test-infra-ref: ${{ github.ref }} + upload-artifact: my-cool-artifact + upload-artifact-to-s3: true + script: | + echo "hello" > "${RUNNER_ARTIFACT_DIR}/cool_beans" test-download-artifact: needs: test-upload-artifact uses: ./.github/workflows/windows_job.yml diff --git a/.github/workflows/windows_job.yml b/.github/workflows/windows_job.yml index fe98ca28c4..d5e03c0000 100644 --- a/.github/workflows/windows_job.yml +++ b/.github/workflows/windows_job.yml @@ -21,6 +21,13 @@ on: under dist/ and any files under artifacts-to-be-uploaded/ will be uploaded default: '' type: string + upload-artifact-to-s3: + description: | + Upload the artifact to S3 instead of GitHub. This is used for large artifacts like + exported model + required: false + default: false + type: boolean download-artifact: description: 'Name to download artifacts to ${RUNNER_ARTIFACT_DIR}' default: '' @@ -185,14 +192,28 @@ jobs: # Set to fail upload step if there are no files for upload and expected files for upload echo 'if-no-files-found=error' >> "${GITHUB_OUTPUT}" + # NB: Keep this for compatibility with existing jobs and also keep in mind that only + # our AWS runners have access to S3 - name: Upload artifacts to GitHub (if any) uses: actions/upload-artifact@v3 - if: ${{ always() && inputs.upload-artifact != '' }} + if: ${{ always() && inputs.upload-artifact != '' && !inputs.upload-artifact-to-s3 }} with: name: ${{ inputs.upload-artifact }} path: ${{ runner.temp }}/artifacts/ if-no-files-found: ${{ steps.check-artifacts.outputs.if-no-files-found }} + # NB: This only works with our AWS runners + - name: Upload artifacts to S3 (if any) + uses: seemethere/upload-artifact-s3@v5 + if: ${{ always() && inputs.upload-artifact != '' && inputs.upload-artifact-to-s3 }} + with: + retention-days: 14 + s3-bucket: gha-artifacts + s3-prefix: | + ${{ env.REPOSITORY }}/${{ github.run_id }}/artifacts + if-no-files-found: ${{ steps.check-artifacts.outputs.if-no-files-found }} + path: ${{ runner.temp }}/artifacts/ + - name: Teardown Windows if: ${{ always() }} uses: ./test-infra/.github/actions/teardown-windows