From eb73a5ecc14b0b8472ca118ec462a49355eb7222 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Mon, 15 Jan 2024 12:17:10 -0800 Subject: [PATCH 1/3] Correctly checkout test-infra on Nova _binary_upload job --- .github/workflows/_binary_upload.yml | 16 ++++++++++++++-- .github/workflows/build_wheels_linux.yml | 2 ++ .github/workflows/build_wheels_macos.yml | 2 ++ .github/workflows/build_wheels_windows.yml | 2 ++ 4 files changed, 20 insertions(+), 2 deletions(-) diff --git a/.github/workflows/_binary_upload.yml b/.github/workflows/_binary_upload.yml index fcc5c0226e..1e4f8c9e21 100644 --- a/.github/workflows/_binary_upload.yml +++ b/.github/workflows/_binary_upload.yml @@ -11,6 +11,14 @@ on: description: 'Reference to checkout, defaults to "nightly"' default: 'nightly' type: string + test-infra-repository: + description: "Test infra repository to use" + default: "pytorch/test-infra" + type: string + test-infra-ref: + description: "Test infra reference to use" + default: "" + type: string build-matrix: description: "Build matrix to utilize" default: '' @@ -36,9 +44,13 @@ jobs: name: ${{ matrix.build_name }} steps: - uses: actions/checkout@v3 + with: + repository: ${{ inputs.test-infra-repository }} + ref: ${{ inputs.test-infra-ref }} + path: test-infra # For pytorch_pkg_helpers which we need to run to generate the artifact name and target S3 buckets - - uses: ./.github/actions/setup-binary-upload + - uses: ./test-infra/.github/actions/setup-binary-upload with: repository: ${{ inputs.repository }} ref: ${{ inputs.ref }} @@ -47,7 +59,7 @@ jobs: arch: ${{ inputs.architecture }} upload-to-base-bucket: ${{ matrix.upload_to_base_bucket }} - - uses: ./.github/actions/set-channel + - uses: ./test-infra/.github/actions/set-channel - name: Download the artifact uses: actions/download-artifact@v3 diff --git a/.github/workflows/build_wheels_linux.yml b/.github/workflows/build_wheels_linux.yml index 0323b098b5..866771e4c9 100644 --- a/.github/workflows/build_wheels_linux.yml +++ b/.github/workflows/build_wheels_linux.yml @@ -235,6 +235,8 @@ jobs: with: repository: ${{ inputs.repository }} ref: ${{ inputs.ref }} + test-infra-repository: ${{ inputs.test-infra-repository }} + test-infra-ref: ${{ inputs.test-infra-ref }} build-matrix: ${{ inputs.build-matrix }} architecture: ${{ inputs.architecture }} trigger-event: ${{ inputs.trigger-event }} diff --git a/.github/workflows/build_wheels_macos.yml b/.github/workflows/build_wheels_macos.yml index 21d52aa9bd..470d872b7b 100644 --- a/.github/workflows/build_wheels_macos.yml +++ b/.github/workflows/build_wheels_macos.yml @@ -214,6 +214,8 @@ jobs: with: repository: ${{ inputs.repository }} ref: ${{ inputs.ref }} + test-infra-repository: ${{ inputs.test-infra-repository }} + test-infra-ref: ${{ inputs.test-infra-ref }} build-matrix: ${{ inputs.build-matrix }} trigger-event: ${{ inputs.trigger-event }} diff --git a/.github/workflows/build_wheels_windows.yml b/.github/workflows/build_wheels_windows.yml index 6df8bbc938..1c4d5ecd03 100644 --- a/.github/workflows/build_wheels_windows.yml +++ b/.github/workflows/build_wheels_windows.yml @@ -202,6 +202,8 @@ jobs: with: repository: ${{ inputs.repository }} ref: ${{ inputs.ref }} + test-infra-repository: ${{ inputs.test-infra-repository }} + test-infra-ref: ${{ inputs.test-infra-ref }} build-matrix: ${{ inputs.build-matrix }} trigger-event: ${{ inputs.trigger-event }} From b2c4505fb6825950290d310297a52c4dc8bc165e Mon Sep 17 00:00:00 2001 From: Huy Do Date: Mon, 15 Jan 2024 12:31:14 -0800 Subject: [PATCH 2/3] Use the correct path to pkg-helpers --- .github/actions/setup-binary-upload/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/setup-binary-upload/action.yml b/.github/actions/setup-binary-upload/action.yml index a45aaf379b..85700d7a06 100644 --- a/.github/actions/setup-binary-upload/action.yml +++ b/.github/actions/setup-binary-upload/action.yml @@ -64,7 +64,7 @@ runs: run: | set -ex - python -m pip install tools/pkg-helpers + python -m pip install ${GITHUB_WORKSPACE}/test-infra/tools/pkg-helpers BUILD_ENV_FILE="${RUNNER_TEMP}/build_env_${GITHUB_RUN_ID}" python -m pytorch_pkg_helpers > "${BUILD_ENV_FILE}" From d535c56a86dafa1409ab2aff26717df007550b64 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Mon, 15 Jan 2024 12:41:17 -0800 Subject: [PATCH 3/3] Need to checkout the target repository to run pkg-helpers --- .github/actions/setup-binary-upload/action.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/actions/setup-binary-upload/action.yml b/.github/actions/setup-binary-upload/action.yml index 85700d7a06..5e24159b4c 100644 --- a/.github/actions/setup-binary-upload/action.yml +++ b/.github/actions/setup-binary-upload/action.yml @@ -53,7 +53,15 @@ runs: # and "/" is not allowed in artifact names echo "ARTIFACT_NAME=${REPOSITORY/\//_}_${REF}_${PYTHON_VERSION}_${CU_VERSION}_${ARCH}" >> "${GITHUB_ENV}" + # Need to checkout the target repository to run pkg-helpers + - uses: actions/checkout@v3 + with: + repository: ${{ inputs.repository }} + ref: ${{ inputs.ref }} + path: ${{ inputs.repository }} + - name: Generate env variables from pytorch_pkg_helpers + working-directory: ${{ inputs.repository }} shell: bash env: REPOSITORY: ${{ inputs.repository }}