From 8946a57d6a92a9f8a8abdfb3dafffae1a15589de Mon Sep 17 00:00:00 2001 From: atalman Date: Wed, 29 Nov 2023 13:15:38 -0800 Subject: [PATCH 1/6] Apply classic solver to M1 builds Add checkchannel where package is installed test test test test test test test test test test test test Fix lint typo --- .github/workflows/build_conda_linux.yml | 24 ++++++++++++++++++++--- .github/workflows/build_conda_macos.yml | 22 +++++++++++++++++++-- .github/workflows/build_conda_windows.yml | 24 ++++++++++++++++++++--- 3 files changed, 62 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build_conda_linux.yml b/.github/workflows/build_conda_linux.yml index 29b40ab1e1..e04bcb676c 100644 --- a/.github/workflows/build_conda_linux.yml +++ b/.github/workflows/build_conda_linux.yml @@ -180,14 +180,32 @@ jobs: CONSTRAINTS="cpuonly" fi + # From Conda documentation: + # https://docs.conda.io/projects/conda/en/latest/user-guide/concepts/channels.html#specifying-channels-when-installing-packages + # From the command line use --c + # You may specify multiple channels by passing the argument multiple times. + # Priority decreases from left to right - the first argument is higher priority than the second. + # From the command line use --override-channels to only search the specified channel(s), + # rather than any channels configured in .condarc. + ${CONDA_RUN_SMOKE} conda install \ --yes \ --quiet \ + -c "${CONDA_LOCAL_CHANNEL}" \ -c pytorch-"${CHANNEL}" \ -c nvidia \ - -c "${CONDA_LOCAL_CHANNEL}" \ - distr::"${PACKAGE_NAME}" \ - ${CONSTRAINTS} + -c defaults \ + "${PACKAGE_NAME}" \ + ${CONSTRAINTS} \ + --override-channels + + # check for installed package channel + CONDA_PKG=$(${CONDA_RUN_SMOKE} conda list | grep "${PACKAGE_NAME}" | grep "${CONDA_LOCAL_CHANNEL}") + if [[ -z "${CONDA_PKG}" ]]; then + ${CONDA_RUN_SMOKE} conda list | grep "${PACKAGE_NAME}" + echo "${PACKAGE_NAME} is not installed from local channel" + exit 1 + fi if [[ ! -f "${{ inputs.repository }}"/${SMOKE_TEST_SCRIPT} ]]; then echo "${{ inputs.repository }}/${SMOKE_TEST_SCRIPT} not found" diff --git a/.github/workflows/build_conda_macos.yml b/.github/workflows/build_conda_macos.yml index 29b99e5b47..a76f13dbd8 100644 --- a/.github/workflows/build_conda_macos.yml +++ b/.github/workflows/build_conda_macos.yml @@ -182,14 +182,32 @@ jobs: export CONDA_RUN_SMOKE="conda run -p ${CONDA_ENV_SMOKE}" export OLD_PATH=${PATH} export PATH="${CONDA_ENV_SMOKE}/bin:${PATH}" - CONDA_LOCAL_CHANNEL="file://$(readlink -f ${{ inputs.repository }}/distr)" + + # From Conda documentation: + # https://docs.conda.io/projects/conda/en/latest/user-guide/concepts/channels.html#specifying-channels-when-installing-packages + # From the command line use --c + # You may specify multiple channels by passing the argument multiple times. + # Priority decreases from left to right - the first argument is higher priority than the second. + # From the command line use --override-channels to only search the specified channel(s), + # rather than any channels configured in .condarc. + ${CONDA_RUN_SMOKE} conda install \ --quiet \ --yes \ -c "${CONDA_LOCAL_CHANNEL}" \ -c pytorch-${CHANNEL} \ - "${PACKAGE_NAME}" + -c defaults \ + "${PACKAGE_NAME}" \ + "--override-channels" + + # check for installed package channel + CONDA_PKG=$(${CONDA_RUN_SMOKE} conda list | grep "${PACKAGE_NAME}" | grep "${CONDA_LOCAL_CHANNEL}") + if [[ -z "${CONDA_PKG}" ]]; then + ${CONDA_RUN_SMOKE} conda list | grep "${PACKAGE_NAME}" + echo "${PACKAGE_NAME} is not installed from local channel" + exit 1 + fi if [[ ! -f "${{ inputs.repository }}"/${SMOKE_TEST_SCRIPT} ]]; then echo "${{ inputs.repository }}/${SMOKE_TEST_SCRIPT} not found" diff --git a/.github/workflows/build_conda_windows.yml b/.github/workflows/build_conda_windows.yml index 4ac0e5d12f..a349c6272e 100644 --- a/.github/workflows/build_conda_windows.yml +++ b/.github/workflows/build_conda_windows.yml @@ -200,14 +200,32 @@ jobs: CONSTRAINTS="cpuonly" fi + # From Conda documentation: + # https://docs.conda.io/projects/conda/en/latest/user-guide/concepts/channels.html#specifying-channels-when-installing-packages + # From the command line use --c + # You may specify multiple channels by passing the argument multiple times. + # Priority decreases from left to right - the first argument is higher priority than the second. + # From the command line use --override-channels to only search the specified channel(s), + # rather than any channels configured in .condarc. + ${CONDA_RUN_SMOKE} conda install \ --quiet \ --yes \ + -c "${CONDA_LOCAL_CHANNEL}" \ -c pytorch-"${CHANNEL}" \ -c nvidia \ - -c "${CONDA_LOCAL_CHANNEL}" \ - distr::"${PACKAGE_NAME}" \ - ${CONSTRAINTS} + -c defaults \ + "${PACKAGE_NAME}" \ + ${CONSTRAINTS} \ + --override-channels + + # check for installed package channel + CONDA_PKG=$(${CONDA_RUN_SMOKE} conda list | grep "${PACKAGE_NAME}" | grep "${CONDA_LOCAL_CHANNEL}") + if [[ -z "${CONDA_PKG}" ]]; then + ${CONDA_RUN_SMOKE} conda list | grep "${PACKAGE_NAME}" + echo "${PACKAGE_NAME} is not installed from local channel" + exit 1 + fi if [[ ! -f "${{ inputs.repository }}"/${SMOKE_TEST_SCRIPT} ]]; then echo "${{ inputs.repository }}/${SMOKE_TEST_SCRIPT} not found" From 7993c56d9026efb45592f74c2aeef999f7b563af Mon Sep 17 00:00:00 2001 From: atalman Date: Wed, 29 Nov 2023 16:16:25 -0800 Subject: [PATCH 2/6] Remove comments --- .github/workflows/build_conda_linux.yml | 8 -------- .github/workflows/build_conda_macos.yml | 8 -------- .github/workflows/build_conda_windows.yml | 8 -------- 3 files changed, 24 deletions(-) diff --git a/.github/workflows/build_conda_linux.yml b/.github/workflows/build_conda_linux.yml index e04bcb676c..fa995a1b5c 100644 --- a/.github/workflows/build_conda_linux.yml +++ b/.github/workflows/build_conda_linux.yml @@ -180,14 +180,6 @@ jobs: CONSTRAINTS="cpuonly" fi - # From Conda documentation: - # https://docs.conda.io/projects/conda/en/latest/user-guide/concepts/channels.html#specifying-channels-when-installing-packages - # From the command line use --c - # You may specify multiple channels by passing the argument multiple times. - # Priority decreases from left to right - the first argument is higher priority than the second. - # From the command line use --override-channels to only search the specified channel(s), - # rather than any channels configured in .condarc. - ${CONDA_RUN_SMOKE} conda install \ --yes \ --quiet \ diff --git a/.github/workflows/build_conda_macos.yml b/.github/workflows/build_conda_macos.yml index a76f13dbd8..349e58a463 100644 --- a/.github/workflows/build_conda_macos.yml +++ b/.github/workflows/build_conda_macos.yml @@ -184,14 +184,6 @@ jobs: export PATH="${CONDA_ENV_SMOKE}/bin:${PATH}" CONDA_LOCAL_CHANNEL="file://$(readlink -f ${{ inputs.repository }}/distr)" - # From Conda documentation: - # https://docs.conda.io/projects/conda/en/latest/user-guide/concepts/channels.html#specifying-channels-when-installing-packages - # From the command line use --c - # You may specify multiple channels by passing the argument multiple times. - # Priority decreases from left to right - the first argument is higher priority than the second. - # From the command line use --override-channels to only search the specified channel(s), - # rather than any channels configured in .condarc. - ${CONDA_RUN_SMOKE} conda install \ --quiet \ --yes \ diff --git a/.github/workflows/build_conda_windows.yml b/.github/workflows/build_conda_windows.yml index a349c6272e..74609a0b0c 100644 --- a/.github/workflows/build_conda_windows.yml +++ b/.github/workflows/build_conda_windows.yml @@ -200,14 +200,6 @@ jobs: CONSTRAINTS="cpuonly" fi - # From Conda documentation: - # https://docs.conda.io/projects/conda/en/latest/user-guide/concepts/channels.html#specifying-channels-when-installing-packages - # From the command line use --c - # You may specify multiple channels by passing the argument multiple times. - # Priority decreases from left to right - the first argument is higher priority than the second. - # From the command line use --override-channels to only search the specified channel(s), - # rather than any channels configured in .condarc. - ${CONDA_RUN_SMOKE} conda install \ --quiet \ --yes \ From 6bfe35bdb5e2065f90491c1abf4b9baee25f55f1 Mon Sep 17 00:00:00 2001 From: atalman Date: Wed, 29 Nov 2023 17:10:07 -0800 Subject: [PATCH 3/6] Use strict priority --- .github/workflows/build_conda_linux.yml | 2 +- .github/workflows/build_conda_macos.yml | 2 +- .github/workflows/build_conda_windows.yml | 10 +--------- 3 files changed, 3 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build_conda_linux.yml b/.github/workflows/build_conda_linux.yml index fa995a1b5c..71b0fd655d 100644 --- a/.github/workflows/build_conda_linux.yml +++ b/.github/workflows/build_conda_linux.yml @@ -189,7 +189,7 @@ jobs: -c defaults \ "${PACKAGE_NAME}" \ ${CONSTRAINTS} \ - --override-channels + --strict-channel-priority # check for installed package channel CONDA_PKG=$(${CONDA_RUN_SMOKE} conda list | grep "${PACKAGE_NAME}" | grep "${CONDA_LOCAL_CHANNEL}") diff --git a/.github/workflows/build_conda_macos.yml b/.github/workflows/build_conda_macos.yml index 349e58a463..30b83fe82f 100644 --- a/.github/workflows/build_conda_macos.yml +++ b/.github/workflows/build_conda_macos.yml @@ -191,7 +191,7 @@ jobs: -c pytorch-${CHANNEL} \ -c defaults \ "${PACKAGE_NAME}" \ - "--override-channels" + "--strict-channel-priority" # check for installed package channel CONDA_PKG=$(${CONDA_RUN_SMOKE} conda list | grep "${PACKAGE_NAME}" | grep "${CONDA_LOCAL_CHANNEL}") diff --git a/.github/workflows/build_conda_windows.yml b/.github/workflows/build_conda_windows.yml index 74609a0b0c..bd2d15500c 100644 --- a/.github/workflows/build_conda_windows.yml +++ b/.github/workflows/build_conda_windows.yml @@ -209,15 +209,7 @@ jobs: -c defaults \ "${PACKAGE_NAME}" \ ${CONSTRAINTS} \ - --override-channels - - # check for installed package channel - CONDA_PKG=$(${CONDA_RUN_SMOKE} conda list | grep "${PACKAGE_NAME}" | grep "${CONDA_LOCAL_CHANNEL}") - if [[ -z "${CONDA_PKG}" ]]; then - ${CONDA_RUN_SMOKE} conda list | grep "${PACKAGE_NAME}" - echo "${PACKAGE_NAME} is not installed from local channel" - exit 1 - fi + --strict-channel-priority if [[ ! -f "${{ inputs.repository }}"/${SMOKE_TEST_SCRIPT} ]]; then echo "${{ inputs.repository }}/${SMOKE_TEST_SCRIPT} not found" From 44e839c3a2ec4931e06d21cab1a9a0bc2575d246 Mon Sep 17 00:00:00 2001 From: atalman Date: Wed, 29 Nov 2023 17:35:38 -0800 Subject: [PATCH 4/6] Revert "Use strict priority" This reverts commit 6bfe35bdb5e2065f90491c1abf4b9baee25f55f1. --- .github/workflows/build_conda_linux.yml | 2 +- .github/workflows/build_conda_macos.yml | 2 +- .github/workflows/build_conda_windows.yml | 10 +++++++++- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_conda_linux.yml b/.github/workflows/build_conda_linux.yml index 71b0fd655d..fa995a1b5c 100644 --- a/.github/workflows/build_conda_linux.yml +++ b/.github/workflows/build_conda_linux.yml @@ -189,7 +189,7 @@ jobs: -c defaults \ "${PACKAGE_NAME}" \ ${CONSTRAINTS} \ - --strict-channel-priority + --override-channels # check for installed package channel CONDA_PKG=$(${CONDA_RUN_SMOKE} conda list | grep "${PACKAGE_NAME}" | grep "${CONDA_LOCAL_CHANNEL}") diff --git a/.github/workflows/build_conda_macos.yml b/.github/workflows/build_conda_macos.yml index 30b83fe82f..349e58a463 100644 --- a/.github/workflows/build_conda_macos.yml +++ b/.github/workflows/build_conda_macos.yml @@ -191,7 +191,7 @@ jobs: -c pytorch-${CHANNEL} \ -c defaults \ "${PACKAGE_NAME}" \ - "--strict-channel-priority" + "--override-channels" # check for installed package channel CONDA_PKG=$(${CONDA_RUN_SMOKE} conda list | grep "${PACKAGE_NAME}" | grep "${CONDA_LOCAL_CHANNEL}") diff --git a/.github/workflows/build_conda_windows.yml b/.github/workflows/build_conda_windows.yml index bd2d15500c..74609a0b0c 100644 --- a/.github/workflows/build_conda_windows.yml +++ b/.github/workflows/build_conda_windows.yml @@ -209,7 +209,15 @@ jobs: -c defaults \ "${PACKAGE_NAME}" \ ${CONSTRAINTS} \ - --strict-channel-priority + --override-channels + + # check for installed package channel + CONDA_PKG=$(${CONDA_RUN_SMOKE} conda list | grep "${PACKAGE_NAME}" | grep "${CONDA_LOCAL_CHANNEL}") + if [[ -z "${CONDA_PKG}" ]]; then + ${CONDA_RUN_SMOKE} conda list | grep "${PACKAGE_NAME}" + echo "${PACKAGE_NAME} is not installed from local channel" + exit 1 + fi if [[ ! -f "${{ inputs.repository }}"/${SMOKE_TEST_SCRIPT} ]]; then echo "${{ inputs.repository }}/${SMOKE_TEST_SCRIPT} not found" From af5018594dc148a872ecb222e0fdff76e6fbc9ef Mon Sep 17 00:00:00 2001 From: atalman Date: Wed, 29 Nov 2023 17:56:08 -0800 Subject: [PATCH 5/6] Add win dbg statement --- .github/workflows/build_conda_windows.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_conda_windows.yml b/.github/workflows/build_conda_windows.yml index 74609a0b0c..4a3c1e9942 100644 --- a/.github/workflows/build_conda_windows.yml +++ b/.github/workflows/build_conda_windows.yml @@ -212,6 +212,7 @@ jobs: --override-channels # check for installed package channel + CONDA_PKG_DBG=$(${CONDA_RUN_SMOKE} conda list | grep "${PACKAGE_NAME}") CONDA_PKG=$(${CONDA_RUN_SMOKE} conda list | grep "${PACKAGE_NAME}" | grep "${CONDA_LOCAL_CHANNEL}") if [[ -z "${CONDA_PKG}" ]]; then ${CONDA_RUN_SMOKE} conda list | grep "${PACKAGE_NAME}" From d5b387dea07717e262c609a7f124f91afa494690 Mon Sep 17 00:00:00 2001 From: atalman Date: Wed, 29 Nov 2023 18:48:01 -0800 Subject: [PATCH 6/6] use_build_version_windows --- .github/workflows/build_conda_windows.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build_conda_windows.yml b/.github/workflows/build_conda_windows.yml index 4a3c1e9942..4ffc9ee4df 100644 --- a/.github/workflows/build_conda_windows.yml +++ b/.github/workflows/build_conda_windows.yml @@ -211,12 +211,11 @@ jobs: ${CONSTRAINTS} \ --override-channels - # check for installed package channel - CONDA_PKG_DBG=$(${CONDA_RUN_SMOKE} conda list | grep "${PACKAGE_NAME}") - CONDA_PKG=$(${CONDA_RUN_SMOKE} conda list | grep "${PACKAGE_NAME}" | grep "${CONDA_LOCAL_CHANNEL}") + # fow windows we use BUILD_VERSION to validate the package + CONDA_PKG=$(${CONDA_RUN_SMOKE} conda list | grep "${PACKAGE_NAME}" | grep "${BUILD_VERSION}") if [[ -z "${CONDA_PKG}" ]]; then - ${CONDA_RUN_SMOKE} conda list | grep "${PACKAGE_NAME}" - echo "${PACKAGE_NAME} is not installed from local channel" + ${CONDA_RUN_SMOKE} conda list | grep "${BUILD_VERSION}" + echo "${PACKAGE_NAME} is not installed with a correct version" exit 1 fi